Compare commits

...
Author SHA1 Message Date
Tim Makram Ghatas 2407b4dd2e remove profiles and implants. Beacons are just removed instead of killing them. 2024-05-31 12:10:49 +02:00
Tim Makram Ghatas ea06daa5ac add clean command and function definitions 2024-05-31 11:46:54 +02:00
Joe d0e5fbb680 Merge pull request #1693 from BishopFox/981ddependabot/go_modules/google.golang.org/grpc-1.64.0
Bump google.golang.org/grpc from 1.63.2 to 1.64.0
2024-05-22 13:48:45 -05:00
dependabot[bot] 6dd56bbfd5 Bump google.golang.org/grpc from 1.63.2 to 1.64.0
Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.63.2 to 1.64.0.
- [Release notes](https://github.com/grpc/grpc-go/releases)
- [Commits](https://github.com/grpc/grpc-go/compare/v1.63.2...v1.64.0)

---
updated-dependencies:
- dependency-name: google.golang.org/grpc
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-22 15:47:44 +00:00
Joe 8f15f896ac Merge pull request #1697 from BishopFox/1071dependabot/go_modules/modernc.org/sqlite-1.29.10
Bump modernc.org/sqlite from 1.29.9 to 1.29.10
2024-05-22 10:46:17 -05:00
Joe a0706ffc1f Merge pull request #1699 from BishopFox/fix_externalbuild_encoderids
send encoder id's to external server during external implant generation
2024-05-22 09:40:30 -05:00
Ronan Kervella 550700de98 Merge branch 'master' into fix_externalbuild_encoderids 2024-05-22 06:49:01 -07:00
dependabot[bot] 652d960d59 Bump modernc.org/sqlite from 1.29.9 to 1.29.10
Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.29.9 to 1.29.10.
- [Commits](https://gitlab.com/cznic/sqlite/compare/v1.29.9...v1.29.10)

---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-21 21:39:28 +00:00
Ronan Kervella 292f41c665 Merge pull request #1696 from BishopFox/b516dependabot/go_modules/github.com/tetratelabs/wazero-1.7.2
Bump github.com/tetratelabs/wazero from 1.7.1 to 1.7.2
2024-05-21 14:32:56 -07:00
dependabot[bot] ef324e8b9d Bump github.com/tetratelabs/wazero from 1.7.1 to 1.7.2
Bumps [github.com/tetratelabs/wazero](https://github.com/tetratelabs/wazero) from 1.7.1 to 1.7.2.
- [Release notes](https://github.com/tetratelabs/wazero/releases)
- [Commits](https://github.com/tetratelabs/wazero/compare/v1.7.1...v1.7.2)

---
updated-dependencies:
- dependency-name: github.com/tetratelabs/wazero
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-21 17:09:18 +00:00
Joe a405ae8f20 Merge pull request #1694 from BishopFox/e220dependabot/go_modules/tailscale.com-1.66.3
Bump tailscale.com from 1.64.2 to 1.66.3
2024-05-21 12:08:00 -05:00
Ronan Kervella 222cc3437b Merge branch 'master' into e220dependabot/go_modules/tailscale.com-1.66.3 2024-05-21 08:35:08 -07:00
Ronan Kervella fdeca972bd Merge pull request #1692 from BishopFox/generate_c2profiles
improve c2profiles command
2024-05-20 15:30:59 -07:00
Tim Makram Ghatas 69bcf72e80 send encoder id's to external server during external implant generation 2024-05-21 00:04:33 +02:00
Timothy Makram Ghatas d60dd6a5a0 Merge branch 'master' into generate_c2profiles 2024-05-20 23:26:37 +02:00
Tim Makram Ghatas 8737bba4c3 enforce unique startsession extensions 2024-05-20 23:25:10 +02:00
Tim Makram Ghatas 97716d2b9e only stager extensions need to be unique 2024-05-20 23:01:38 +02:00
Tim Makram Ghatas 844a6f8d25 check there are sufficient unused extensions in the provided file 2024-05-20 22:58:09 +02:00
Tim Makram Ghatas b6f2c1b4bb fix documentation for generate command 2024-05-20 22:23:56 +02:00
Ronan Kervella c6397918c6 Merge branch 'master' into e220dependabot/go_modules/tailscale.com-1.66.3 2024-05-20 11:02:00 -07:00
Ronan Kervella 848704eb3c Merge pull request #1695 from BishopFox/0bafdependabot/go_modules/github.com/fatih/color-1.17.0
Bump github.com/fatih/color from 1.16.0 to 1.17.0
2024-05-20 11:01:48 -07:00
dependabot[bot] c3571b001a Bump github.com/fatih/color from 1.16.0 to 1.17.0
Bumps [github.com/fatih/color](https://github.com/fatih/color) from 1.16.0 to 1.17.0.
- [Release notes](https://github.com/fatih/color/releases)
- [Commits](https://github.com/fatih/color/compare/v1.16.0...v1.17.0)

---
updated-dependencies:
- dependency-name: github.com/fatih/color
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-20 03:37:24 +00:00
dependabot[bot] 53449c1a83 Bump tailscale.com from 1.64.2 to 1.66.3
Bumps [tailscale.com](https://github.com/tailscale/tailscale) from 1.64.2 to 1.66.3.
- [Release notes](https://github.com/tailscale/tailscale/releases)
- [Commits](https://github.com/tailscale/tailscale/compare/v1.64.2...v1.66.3)

---
updated-dependencies:
- dependency-name: tailscale.com
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-20 03:37:15 +00:00
Tim Makram Ghatas cf6682dff7 allow generating c2profiles using a file containing one url per line. Fix autocomplete for export c2profile name 2024-05-19 13:18:26 +02:00
Joe 2a1453ee37 Merge pull request #1691 from BishopFox/v1.6.0/certificate-management
View Certificate Information
2024-05-17 12:46:13 -05:00
Raf a9f06fc299 Merge branch 'master' into v1.6.0/certificate-management 2024-05-17 09:39:30 -04:00
Raf 5839c59682 Changing help text for certificates command 2024-05-17 09:35:54 -04:00
Raf 860810c148 Adding filters to certificate command 2024-05-17 09:33:24 -04:00
Ronan Kervella d92f079f97 Merge pull request #1690 from w33ts/patch-1
Fixed command typo in Getting Started.md
2024-05-17 13:19:58 +02:00
w33ts 4315a3fce3 Fixed command typo in Getting Started.md
There is a command that says "systemctrl" instead of "systemctl". Yup, I'm *that* guy now.

Signed-off-by: w33ts <[email protected]>
2024-05-16 21:50:34 -05:00
Joe 5d142acb86 Merge pull request #1681 from BishopFox/1c54dependabot/go_modules/golang.org/x/net-0.25.0
Bump golang.org/x/net from 0.24.0 to 0.25.0
2024-05-16 14:43:36 -05:00
Raf 0ee50c3150 Adding ID to certificates output, adjusting for narrow terms 2024-05-16 13:37:30 -04:00
Joe c0840eb9d6 Merge branch 'master' into 1c54dependabot/go_modules/golang.org/x/net-0.25.0 2024-05-16 10:54:09 -05:00
Joe 8b3e3cd6ab Merge pull request #1682 from BishopFox/5a34dependabot/go_modules/github.com/cheggaaa/pb/v3-3.1.5
Bump github.com/cheggaaa/pb/v3 from 3.1.2 to 3.1.5
2024-05-16 10:53:39 -05:00
Joe ffd2fa6e8b Merge pull request #1686 from BishopFox/fix-install-script
Fixing install script for Arch
2024-05-16 10:53:23 -05:00
Joe 130b5f9706 Merge pull request #1687 from BishopFox/v1.6.0/database-implant-table-fixes
Improving PostgreSQL Support
2024-05-16 10:53:06 -05:00
Raf 549a54cf4b Initial components of the certificates command
This commit adds a command (certificates) to the client that retrieves selected certificate information from the database.
2024-05-16 11:32:34 -04:00
Raf e2d11c40dd Fixed poor declaration of DB models for migration 2024-05-14 11:49:44 -04:00
Raf 9749cd8e58 Merge branch 'master' into v1.6.0/database-implant-table-fixes 2024-05-14 10:42:22 -04:00
Raf 4082378313 Adding nil check to listener deletion to prevent panic 2024-05-14 10:23:42 -04:00
Raf df65d7fe11 More thorough deletion of jobs from the DB when they are killed 2024-05-14 10:22:00 -04:00
Raf 99ce1cfb7a Fixing FK constraint while deleting profiles 2024-05-14 09:07:06 -04:00
Raf 5f2abac8d0 Database model fixes for non-SQLite databases, part 2
Changing implant generation without an associated profile to insert NULL into the database instead of the zero GUID
2024-05-13 15:43:04 -04:00
dependabot[bot] f265214006 Bump github.com/cheggaaa/pb/v3 from 3.1.2 to 3.1.5
Bumps [github.com/cheggaaa/pb/v3](https://github.com/cheggaaa/pb) from 3.1.2 to 3.1.5.
- [Commits](https://github.com/cheggaaa/pb/compare/v3.1.2...v3.1.5)

---
updated-dependencies:
- dependency-name: github.com/cheggaaa/pb/v3
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-13 17:14:40 +00:00
dependabot[bot] 23bdc7053d Bump golang.org/x/net from 0.24.0 to 0.25.0
Bumps [golang.org/x/net](https://github.com/golang/net) from 0.24.0 to 0.25.0.
- [Commits](https://github.com/golang/net/compare/v0.24.0...v0.25.0)

---
updated-dependencies:
- dependency-name: golang.org/x/net
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-13 17:14:34 +00:00
Joe 9510a5f0c1 Merge pull request #1685 from BishopFox/47b6dependabot/go_modules/google.golang.org/protobuf-1.34.1
Bump google.golang.org/protobuf from 1.34.0 to 1.34.1
2024-05-13 12:13:32 -05:00
Joe 72ea11790f Merge pull request #1683 from BishopFox/e97ddependabot/go_modules/golang.org/x/crypto-0.23.0
Bump golang.org/x/crypto from 0.22.0 to 0.23.0
2024-05-13 12:13:19 -05:00
Raf d651e0888f Database model fixes for non-SQLite databases, part 1
Changing database migrations to occur in sequence instead of all at once for more accurate error reporting in the log
Changing HostUUID in the models.Host struct to be unique to fix a constraint issue
Changing models.HttpC2Header to contain pointers to ServerConfigIDs and ImplantConfigIDs so that NULL values can be inserted into the database when a type of config does not apply.
2024-05-13 12:55:53 -04:00
Raf f2bb628883 Fixing install script for Arch 2024-05-13 09:15:20 -04:00
dependabot[bot] 8e5ff45306 Bump google.golang.org/protobuf from 1.34.0 to 1.34.1
Bumps google.golang.org/protobuf from 1.34.0 to 1.34.1.

---
updated-dependencies:
- dependency-name: google.golang.org/protobuf
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-13 08:03:08 +00:00
Ronan Kervella 9dbd5a1c92 Merge branch 'master' into e97ddependabot/go_modules/golang.org/x/crypto-0.23.0 2024-05-13 10:02:18 +02:00
Ronan Kervella 6115702268 Merge pull request #1684 from BishopFox/527ddependabot/go_modules/modernc.org/sqlite-1.29.9
Bump modernc.org/sqlite from 1.29.8 to 1.29.9
2024-05-13 10:01:48 +02:00
dependabot[bot] ed1739c8d2 Bump modernc.org/sqlite from 1.29.8 to 1.29.9
Bumps [modernc.org/sqlite](https://gitlab.com/cznic/sqlite) from 1.29.8 to 1.29.9.
- [Commits](https://gitlab.com/cznic/sqlite/compare/v1.29.8...v1.29.9)

---
updated-dependencies:
- dependency-name: modernc.org/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-13 03:08:40 +00:00
dependabot[bot] 88da1fe868 Bump golang.org/x/crypto from 0.22.0 to 0.23.0
Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.22.0 to 0.23.0.
- [Commits](https://github.com/golang/crypto/compare/v0.22.0...v0.23.0)

---
updated-dependencies:
- dependency-name: golang.org/x/crypto
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-13 03:08:17 +00:00
Joe b520eaf6e6 Merge pull request #1679 from BishopFox/v1.6.0/fix-console-hang
Fix #1668
2024-05-10 09:31:13 -05:00
rkervella 635086364e Fix #1668 2024-05-10 11:46:31 +02:00
Ronan Kervella 4bfdb7bcee Merge pull request #1676 from BishopFox/v1.6.0/package-output
New Output Handler for Extensions and Aliases
2024-05-08 09:46:49 +02:00
Ronan Kervella 7e4ba5d3e7 Merge branch 'master' into v1.6.0/package-output 2024-05-07 17:20:47 +02:00
Ronan Kervella c642bfcacb Merge pull request #1675 from BishopFox/xforwardedfor_fix
extract original host ip if forwarded by one or more hosts
2024-05-07 17:20:17 +02:00
Timothy Makram Ghatas a424412edd Merge branch 'master' into xforwardedfor_fix 2024-05-07 15:34:08 +02:00
Raf 4d15774c9b Merge branch 'master' into v1.6.0/package-output 2024-05-07 09:05:14 -04:00
Raf 51c45a48e8 Adding error handling for bad ingestion 2024-05-07 09:04:55 -04:00
Tim Makram Ghatas 94f01c8230 added default case if only one forwarding ip exists 2024-05-07 14:41:51 +02:00
Raf 0307bf16eb Adding a new output handler for extensions and aliases that deserializes JSON output into tables 2024-05-07 08:30:17 -04:00
Tim Makram Ghatas 1a73128163 extract original host ip if forwarded by one or more hosts 2024-05-07 12:31:01 +02:00
Joe eff371f6b7 Merge pull request #1671 from BishopFox/d2cbdependabot/go_modules/github.com/stretchr/testify-1.9.0
Bump github.com/stretchr/testify from 1.8.4 to 1.9.0
2024-05-06 10:37:28 -05:00
Joe 3a48d8480e Merge pull request #1674 from BishopFox/f9f5dependabot/go_modules/golang.org/x/text-0.15.0
Bump golang.org/x/text from 0.14.0 to 0.15.0
2024-05-06 10:37:16 -05:00
Joe 5cd83392e0 Merge pull request #1672 from BishopFox/7227dependabot/go_modules/github.com/glebarez/sqlite-1.11.0
Bump github.com/glebarez/sqlite from 1.10.0 to 1.11.0
2024-05-06 10:37:06 -05:00
Joe fffaca3945 Merge pull request #1670 from BishopFox/7890dependabot/go_modules/github.com/ncruces/go-sqlite3-0.15.0
Bump github.com/ncruces/go-sqlite3 from 0.7.2 to 0.15.0
2024-05-06 10:36:54 -05:00
dependabot[bot] fae797eff0 Bump github.com/ncruces/go-sqlite3 from 0.7.2 to 0.15.0
Bumps [github.com/ncruces/go-sqlite3](https://github.com/ncruces/go-sqlite3) from 0.7.2 to 0.15.0.
- [Release notes](https://github.com/ncruces/go-sqlite3/releases)
- [Commits](https://github.com/ncruces/go-sqlite3/compare/v0.7.2...v0.15.0)

---
updated-dependencies:
- dependency-name: github.com/ncruces/go-sqlite3
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-06 08:14:52 +00:00
dependabot[bot] 64a7095f5f Bump github.com/stretchr/testify from 1.8.4 to 1.9.0
Bumps [github.com/stretchr/testify](https://github.com/stretchr/testify) from 1.8.4 to 1.9.0.
- [Release notes](https://github.com/stretchr/testify/releases)
- [Commits](https://github.com/stretchr/testify/compare/v1.8.4...v1.9.0)

---
updated-dependencies:
- dependency-name: github.com/stretchr/testify
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-06 08:13:52 +00:00
dependabot[bot] 82909ad1d9 Bump github.com/glebarez/sqlite from 1.10.0 to 1.11.0
Bumps [github.com/glebarez/sqlite](https://github.com/glebarez/sqlite) from 1.10.0 to 1.11.0.
- [Release notes](https://github.com/glebarez/sqlite/releases)
- [Commits](https://github.com/glebarez/sqlite/compare/v1.10.0...v1.11.0)

---
updated-dependencies:
- dependency-name: github.com/glebarez/sqlite
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-06 08:13:37 +00:00
Ronan Kervella e2e9bc18dc Merge branch 'master' into f9f5dependabot/go_modules/golang.org/x/text-0.15.0 2024-05-06 01:12:43 -07:00
Ronan Kervella 5bed9d4e90 Merge pull request #1673 from BishopFox/c6e8dependabot/go_modules/github.com/jedib0t/go-pretty/v6-6.5.9
Bump github.com/jedib0t/go-pretty/v6 from 6.5.8 to 6.5.9
2024-05-06 01:12:20 -07:00
dependabot[bot] 60e87f9e08 Bump golang.org/x/text from 0.14.0 to 0.15.0
Bumps [golang.org/x/text](https://github.com/golang/text) from 0.14.0 to 0.15.0.
- [Release notes](https://github.com/golang/text/releases)
- [Commits](https://github.com/golang/text/compare/v0.14.0...v0.15.0)

---
updated-dependencies:
- dependency-name: golang.org/x/text
  dependency-type: direct:production
  update-type: version-update:semver-minor
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-06 03:45:35 +00:00
dependabot[bot] d4c37f45d2 Bump github.com/jedib0t/go-pretty/v6 from 6.5.8 to 6.5.9
Bumps [github.com/jedib0t/go-pretty/v6](https://github.com/jedib0t/go-pretty) from 6.5.8 to 6.5.9.
- [Release notes](https://github.com/jedib0t/go-pretty/releases)
- [Commits](https://github.com/jedib0t/go-pretty/compare/v6.5.8...v6.5.9)

---
updated-dependencies:
- dependency-name: github.com/jedib0t/go-pretty/v6
  dependency-type: direct:production
  update-type: version-update:semver-patch
...

Signed-off-by: dependabot[bot] <[email protected]>
2024-05-06 03:45:25 +00:00
Joe 79d22bafbf Merge pull request #1669 from ngn13/wg-config-fix
fix: add missing newline
2024-05-05 10:56:55 -05:00
ngn 6396a64878 add missing newline
Signed-off-by: ngn <[email protected]>
2024-05-04 23:41:34 +00:00
Joe 335797e661 Merge pull request #1667 from BishopFox/docs/community-guides
Docs/community guides
2024-05-03 21:13:39 -05:00
moloch-- d767bd019c Add youtube embed component 2024-05-03 19:13:02 -07:00
moloch-- 590f9c3f96 Add youtube embed component 2024-05-03 18:59:42 -07:00
Joe cff899aa49 Merge pull request #1646 from testwill/close_resource
fix: close resource
2024-05-03 12:17:46 -05:00
Joe ca2dc9d267 Merge pull request #1665 from BishopFox/c2profile_update_bugfix
C2profile update bugfix
2024-05-03 12:17:19 -05:00
Ronan Kervella afdd77c642 Merge branch 'master' into close_resource 2024-05-03 09:21:54 -07:00
Ronan Kervella 8a5970c3da Merge branch 'master' into c2profile_update_bugfix 2024-05-03 09:21:40 -07:00
Ronan Kervella 2eb2ac8121 Merge pull request #1659 from C-Sto/bugfix-extensions
fix extension registration, run, and load flows
2024-05-03 09:21:29 -07:00
Ronan Kervella daf7d404b1 Merge branch 'master' into c2profile_update_bugfix 2024-05-03 09:21:23 -07:00
Ronan Kervella 3b308920f4 Merge branch 'master' into close_resource 2024-05-03 08:34:23 -07:00
Ronan Kervella c73b183f74 Merge branch 'master' into bugfix-extensions 2024-05-03 08:32:44 -07:00
Ronan Kervella 7564312f5e Merge pull request #1666 from BishopFox/v1.6.0/fix-alias-hostproc
Fix Default Linux Host Process for Aliases
2024-05-03 08:32:22 -07:00
Raf 6134b78afb Fix default linux host process for aliases 2024-05-03 08:24:32 -04:00
Ronan Kervella 3db9946608 Merge branch 'master' into bugfix-extensions 2024-05-02 14:11:02 -07:00
Joe ef09058498 Merge pull request #1664 from BishopFox/v1.6.0/ls-rootdir
Adding root directory entry to `ls` command
2024-05-02 16:06:46 -05:00
Tim Makram Ghatas 331faeb8ef fix commandline paramameters for profile staging 2024-05-02 20:30:33 +02:00
Tim Makram Ghatas 4ae195e3eb fix c2 profile update issue with headers, cookies and segments 2024-05-02 19:09:46 +02:00
Ronan Kervella f8f8ba5864 Merge branch 'master' into bugfix-extensions 2024-05-02 09:59:25 -07:00
Raf 216e3b9759 Adds an entry in the directory listing returned by ls for the root directory ('.'). 2024-05-02 12:12:32 -04:00
Joe f1bc5074f9 Merge pull request #1663 from BishopFox/docs/compile-from-source
Add to index
2024-05-01 21:40:10 -05:00
moloch-- 5083faa1e1 Add to index 2024-05-01 19:39:34 -07:00
Joe 1b5d3436eb Merge pull request #1662 from BishopFox/docs/compile-from-source
Improve compile from source docs
2024-05-01 21:23:20 -05:00
moloch-- 56303eb89a Improve compile from source docs 2024-05-01 19:22:57 -07:00
Joe 9397fcb4ae Merge pull request #1661 from BishopFox/fix_stage_encoder
fix merge error for stage encoders
2024-05-01 20:11:35 -05:00
Joe 04c6c3f800 Merge pull request #1658 from C-Sto/bugfix-extensionnoargs
fix: handle noargs case
2024-05-01 20:08:17 -05:00
C-Sto 7842a6c4c6 add init back in, handle nils/zero values more elegantly 2024-05-02 08:49:21 +08:00
Tim Makram Ghatas 6c815a4432 fix merge error for stage encoders 2024-05-02 02:06:54 +02:00
C-Sto 81358e31bb fix extension registration, run, and load flows 2024-05-01 12:31:19 +08:00
C-Sto 7bea772dbd fix: handle noargs case 2024-05-01 12:15:33 +08:00
Joe 0b235b4d42 Merge pull request #1656 from BishopFox/fix/deps
Update Various Server and Implant Dependencies
2024-04-30 16:33:11 -07:00
Joe eddfb7073c Merge pull request #1655 from BishopFox/fix/init-logging
Fix/init logging
2024-04-30 14:58:00 -07:00
guangwu 7fe0b295f5 fix: close resource
Signed-off-by: guangwu <[email protected]>
2024-04-25 20:54:30 +08:00
518 changed files with 91028 additions and 20826 deletions

No files matched your search

+19
View File
@@ -19,7 +19,9 @@ package cli
*/
import (
"context"
"fmt"
"os"
"github.com/bishopfox/sliver/client/assets"
"github.com/bishopfox/sliver/client/command"
@@ -28,6 +30,7 @@ import (
"github.com/bishopfox/sliver/protobuf/rpcpb"
"github.com/spf13/cobra"
"google.golang.org/grpc"
"google.golang.org/grpc/connectivity"
)
// consoleCmd generates the console with required pre/post runners.
@@ -70,6 +73,9 @@ func consoleRunnerCmd(con *console.SliverClient, run bool) (pre, post func(cmd *
return nil
}
// Wait for any connection state changes and exit if the connection is lost.
go handleConnectionLost(ln)
return console.StartClient(con, rpc, command.ServerCommands(con, nil), command.SliverCommands(con), run)
}
@@ -84,3 +90,16 @@ func consoleRunnerCmd(con *console.SliverClient, run bool) (pre, post func(cmd *
return pre, post
}
func handleConnectionLost(ln *grpc.ClientConn) {
currentState := ln.GetState()
// currentState should be "Ready" when the connection is established.
if ln.WaitForStateChange(context.Background(), currentState) {
newState := ln.GetState()
// newState will be "Idle" if the connection is lost.
if newState == connectivity.Idle {
fmt.Println("\nLost connection to server. Exiting now.")
os.Exit(1)
}
}
}
+72 -19
View File
@@ -32,6 +32,7 @@ import (
"github.com/bishopfox/sliver/client/command/help"
"github.com/bishopfox/sliver/client/console"
consts "github.com/bishopfox/sliver/client/constants"
"github.com/bishopfox/sliver/client/packages"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
"github.com/bishopfox/sliver/util"
@@ -57,7 +58,7 @@ var (
defaultHostProc = map[string]string{
"windows": windowsDefaultHostProc,
"linux": windowsDefaultHostProc,
"linux": linuxDefaultHostProc,
"darwin": macosDefaultHostProc,
}
)
@@ -85,12 +86,13 @@ type AliasManifest struct {
Help string `json:"help"`
LongHelp string `json:"long_help"`
Entrypoint string `json:"entrypoint"`
AllowArgs bool `json:"allow_args"`
DefaultArgs string `json:"default_args"`
Files []*AliasFile `json:"files"`
IsReflective bool `json:"is_reflective"`
IsAssembly bool `json:"is_assembly"`
Entrypoint string `json:"entrypoint"`
AllowArgs bool `json:"allow_args"`
DefaultArgs string `json:"default_args"`
Files []*AliasFile `json:"files"`
IsReflective bool `json:"is_reflective"`
IsAssembly bool `json:"is_assembly"`
Schema *packages.OutputSchema `json:"schema"`
RootPath string `json:"-"`
ArmoryName string `json:"-"`
@@ -239,6 +241,13 @@ func ParseAliasManifest(data []byte) (*AliasManifest, error) {
}
}
if alias.Schema != nil {
if !packages.IsValidSchemaType(alias.Schema.Name) {
return nil, fmt.Errorf("%s is not a valid schema type", alias.Schema.Name)
}
alias.Schema.IngestColumns()
}
return alias, nil
}
@@ -387,11 +396,11 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
con.PrintErrorf("Failed to decode call ext response %s\n", err)
return
}
PrintAssemblyOutput(cmd.Name(), executeAssemblyResp, outFilePath, con)
PrintAssemblyOutput(cmd.Name(), aliasManifest.Schema, executeAssemblyResp, outFilePath, con)
})
con.PrintAsyncResponse(executeAssemblyResp.Response)
} else {
PrintAssemblyOutput(cmd.Name(), executeAssemblyResp, outFilePath, con)
PrintAssemblyOutput(cmd.Name(), aliasManifest.Schema, executeAssemblyResp, outFilePath, con)
}
} else if aliasManifest.IsReflective {
@@ -426,11 +435,11 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
con.PrintErrorf("Failed to decode call ext response %s\n", err)
return
}
PrintSpawnDLLOutput(cmd.Name(), spawnDllResp, outFilePath, con)
PrintSpawnDLLOutput(cmd.Name(), aliasManifest.Schema, spawnDllResp, outFilePath, con)
})
con.PrintAsyncResponse(spawnDllResp.Response)
} else {
PrintSpawnDLLOutput(cmd.Name(), spawnDllResp, outFilePath, con)
PrintSpawnDLLOutput(cmd.Name(), aliasManifest.Schema, spawnDllResp, outFilePath, con)
}
} else {
@@ -466,18 +475,44 @@ func runAliasCommand(cmd *cobra.Command, con *console.SliverClient, args []strin
con.PrintErrorf("Failed to decode call ext response %s\n", err)
return
}
PrintSideloadOutput(cmd.Name(), sideloadResp, outFilePath, con)
PrintSideloadOutput(cmd.Name(), aliasManifest.Schema, sideloadResp, outFilePath, con)
})
con.PrintAsyncResponse(sideloadResp.Response)
} else {
PrintSideloadOutput(cmd.Name(), sideloadResp, outFilePath, con)
PrintSideloadOutput(cmd.Name(), aliasManifest.Schema, sideloadResp, outFilePath, con)
}
}
}
func getOutputWithSchema(schema *packages.OutputSchema, result string) string {
if schema == nil {
return result
}
outputSchema := packages.GetNewPackageOutput(schema.Name)
if outputSchema == nil {
return result
}
err := outputSchema.IngestData([]byte(result), schema.Columns(), schema.GroupBy)
if err != nil {
return result
}
return outputSchema.CreateTable()
}
// PrintSpawnDLLOutput - Prints the output of a spawn dll command.
func PrintSpawnDLLOutput(cmdName string, spawnDllResp *sliverpb.SpawnDll, outFilePath *os.File, con *console.SliverClient) {
con.PrintInfof("%s output:\n%s", cmdName, spawnDllResp.GetResult())
func PrintSpawnDLLOutput(cmdName string, schema *packages.OutputSchema, spawnDllResp *sliverpb.SpawnDll, outFilePath *os.File, con *console.SliverClient) {
var result string
if schema != nil {
result = getOutputWithSchema(schema, spawnDllResp.GetResult())
} else {
result = spawnDllResp.GetResult()
}
con.PrintInfof("%s output:\n%s", cmdName, result)
// Output the raw result to the file
if outFilePath != nil {
outFilePath.WriteString(spawnDllResp.GetResult())
con.PrintInfof("Output saved to %s\n", outFilePath.Name())
@@ -485,8 +520,17 @@ func PrintSpawnDLLOutput(cmdName string, spawnDllResp *sliverpb.SpawnDll, outFil
}
// PrintSideloadOutput - Prints the output of a sideload command.
func PrintSideloadOutput(cmdName string, sideloadResp *sliverpb.Sideload, outFilePath *os.File, con *console.SliverClient) {
con.PrintInfof("%s output:\n%s", cmdName, sideloadResp.GetResult())
func PrintSideloadOutput(cmdName string, schema *packages.OutputSchema, sideloadResp *sliverpb.Sideload, outFilePath *os.File, con *console.SliverClient) {
var result string
if schema != nil {
result = getOutputWithSchema(schema, sideloadResp.GetResult())
} else {
result = sideloadResp.GetResult()
}
con.PrintInfof("%s output:\n%s", cmdName, result)
// Output the raw result to the file
if outFilePath != nil {
outFilePath.WriteString(sideloadResp.GetResult())
con.PrintInfof("Output saved to %s\n", outFilePath.Name())
@@ -494,8 +538,17 @@ func PrintSideloadOutput(cmdName string, sideloadResp *sliverpb.Sideload, outFil
}
// PrintAssemblyOutput - Prints the output of an execute-assembly command.
func PrintAssemblyOutput(cmdName string, execAsmResp *sliverpb.ExecuteAssembly, outFilePath *os.File, con *console.SliverClient) {
con.PrintInfof("%s output:\n%s", cmdName, string(execAsmResp.GetOutput()))
func PrintAssemblyOutput(cmdName string, schema *packages.OutputSchema, execAsmResp *sliverpb.ExecuteAssembly, outFilePath *os.File, con *console.SliverClient) {
var result string
if schema != nil {
result = getOutputWithSchema(schema, string(execAsmResp.GetOutput()))
} else {
result = string(execAsmResp.GetOutput())
}
con.PrintInfof("%s output:\n%s", cmdName, result)
// Output the raw result to the file
if outFilePath != nil {
outFilePath.Write(execAsmResp.GetOutput())
con.PrintInfof("Output saved to %s\n", outFilePath.Name())
+214 -8
View File
@@ -21,8 +21,13 @@ package c2profiles
import (
"context"
"encoding/json"
"fmt"
"io"
"math/rand"
"net/url"
"os"
"path"
"path/filepath"
"strings"
"github.com/AlecAivazis/survey/v2"
@@ -154,7 +159,13 @@ func ExportC2ProfileCmd(cmd *cobra.Command, con *console.SliverClient, args []st
return
}
jsonProfile, err := C2ConfigToJSON(profileName, profile)
config, err := C2ConfigToJSON(profileName, profile)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
jsonProfile, err := json.Marshal(config)
if err != nil {
con.PrintErrorf("%s\n", err)
return
@@ -169,8 +180,96 @@ func ExportC2ProfileCmd(cmd *cobra.Command, con *console.SliverClient, args []st
con.Println(profileName, "C2 profile exported to ", filepath)
}
func GenerateC2ProfileCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
// load template to use as starting point
template, err := cmd.Flags().GetString("template")
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
profileName, _ := cmd.Flags().GetString("name")
if profileName == "" {
con.PrintErrorf("Invalid c2 profile name\n")
return
}
profile, err := con.Rpc.GetHTTPC2ProfileByName(context.Background(), &clientpb.C2ProfileReq{Name: template})
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
c2Profiles, err := con.Rpc.GetHTTPC2Profiles(context.Background(), &commonpb.Empty{})
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
var extensions []string
for _, c2profile := range c2Profiles.Configs {
confProfile, err := con.Rpc.GetHTTPC2ProfileByName(context.Background(), &clientpb.C2ProfileReq{Name: c2profile.Name})
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
extensions = append(extensions, confProfile.ImplantConfig.StagerFileExtension)
extensions = append(extensions, confProfile.ImplantConfig.StartSessionFileExtension)
}
config, err := C2ConfigToJSON(profileName, profile)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
// read urls files and replace segments
filepath, err := cmd.Flags().GetString("file")
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
urlsFile, err := os.Open(filepath)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
fileContent, err := io.ReadAll(urlsFile)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
urls := strings.Split(string(fileContent), "\n")
jsonProfile, err := updateC2Profile(extensions, config, urls)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
// save or display config
importC2Profile, err := cmd.Flags().GetBool("import")
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
if importC2Profile {
httpC2ConfigReq := clientpb.HTTPC2ConfigReq{C2Config: C2ConfigToProtobuf(profileName, jsonProfile)}
_, err = con.Rpc.SaveHTTPC2Profile(context.Background(), &httpC2ConfigReq)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
con.Println("C2 profile generated and saved as ", profileName)
} else {
PrintC2Profiles(profile, con)
}
}
// convert protobuf to json
func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) ([]byte, error) {
func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) (*assets.HTTPC2Config, error) {
implantConfig := assets.HTTPC2ImplantConfig{
UserAgent: profile.ImplantConfig.UserAgent,
ChromeBaseVersion: int(profile.ImplantConfig.ChromeBaseVersion),
@@ -278,12 +377,7 @@ func C2ConfigToJSON(profileName string, profile *clientpb.HTTPC2Config) ([]byte,
ServerConfig: serverConfig,
}
jsonConfig, err := json.Marshal(config)
if err != nil {
return nil, err
}
return jsonConfig, nil
return &config, nil
}
// convert json to protobuf
@@ -598,3 +692,115 @@ func selectC2Profile(c2profiles []*clientpb.HTTPC2Config) string {
return c2profile
}
func updateC2Profile(usedExtensions []string, template *assets.HTTPC2Config, urls []string) (*assets.HTTPC2Config, error) {
// update the template with the urls
var (
paths []string
filenames []string
extensions []string
filteredExtensions []string
)
for _, urlPath := range urls {
parsedURL, err := url.Parse(urlPath)
if err != nil {
fmt.Println("Error parsing URL:", err)
continue
}
dir, file := path.Split(parsedURL.Path)
dir = strings.Trim(dir, "/")
if dir != "" {
paths = append(paths, strings.Split(dir, "/")...)
}
if file != "" {
fileName := strings.TrimSuffix(file, filepath.Ext(file))
filenames = append(filenames, fileName)
ext := strings.TrimPrefix(filepath.Ext(file), ".")
if ext != "" {
extensions = append(extensions, ext)
}
}
}
slices.Sort(extensions)
extensions = slices.Compact(extensions)
for _, extension := range extensions {
if !slices.Contains(usedExtensions, extension) {
filteredExtensions = append(filteredExtensions, extension)
}
}
slices.Sort(paths)
paths = slices.Compact(paths)
slices.Sort(filenames)
filenames = slices.Compact(filenames)
// 5 is arbitrarily used as a minimum value, it only has to be 5 for the extensions, the others can be lower
if len(filteredExtensions) < 5 {
return nil, fmt.Errorf("got %d unused extensions, need at least 5", len(filteredExtensions))
}
if len(paths) < 5 {
return nil, fmt.Errorf("got %d paths need at least 5", len(paths))
}
if len(filenames) < 5 {
return nil, fmt.Errorf("got %d paths need at least 5", len(filenames))
}
// shuffle extensions
for i := len(extensions) - 1; i > 0; i-- {
j := rand.Intn(i + 1)
extensions[i], extensions[j] = extensions[j], extensions[i]
}
template.ImplantConfig.PollFileExt = extensions[0]
template.ImplantConfig.StagerFileExt = extensions[1]
template.ImplantConfig.StartSessionFileExt = extensions[2]
template.ImplantConfig.SessionFileExt = extensions[3]
template.ImplantConfig.CloseFileExt = extensions[4]
// randomly distribute the paths and filenames into the different segment types
template.ImplantConfig.CloseFiles = []string{}
template.ImplantConfig.SessionFiles = []string{}
template.ImplantConfig.PollFiles = []string{}
template.ImplantConfig.StagerFiles = []string{}
template.ImplantConfig.ClosePaths = []string{}
template.ImplantConfig.SessionPaths = []string{}
template.ImplantConfig.PollPaths = []string{}
template.ImplantConfig.StagerPaths = []string{}
for _, path := range paths {
switch rand.Intn(4) {
case 0:
template.ImplantConfig.PollPaths = append(template.ImplantConfig.PollPaths, path)
case 1:
template.ImplantConfig.SessionPaths = append(template.ImplantConfig.SessionPaths, path)
case 2:
template.ImplantConfig.ClosePaths = append(template.ImplantConfig.ClosePaths, path)
case 3:
template.ImplantConfig.StagerPaths = append(template.ImplantConfig.StagerPaths, path)
}
}
for _, filename := range filenames {
switch rand.Intn(4) {
case 0:
template.ImplantConfig.PollFiles = append(template.ImplantConfig.PollFiles, filename)
case 1:
template.ImplantConfig.SessionFiles = append(template.ImplantConfig.SessionFiles, filename)
case 2:
template.ImplantConfig.CloseFiles = append(template.ImplantConfig.CloseFiles, filename)
case 3:
template.ImplantConfig.StagerFiles = append(template.ImplantConfig.StagerFiles, filename)
}
}
return template, nil
}
+25 -1
View File
@@ -39,7 +39,9 @@ func Commands(con *console.SliverClient) []*cobra.Command {
flags.Bind(consts.ExportC2ProfileStr, false, exportC2ProfileCmd, func(f *pflag.FlagSet) {
f.StringP("file", "f", "", "Path to file to export C2 configuration to")
f.StringP("name", "n", consts.DefaultC2Profile, "HTTP C2 Profile name")
})
flags.BindFlagCompletions(exportC2ProfileCmd, func(comp *carapace.ActionMap) {
(*comp)["name"] = generate.HTTPC2Completer(con)
})
C2ProfileCmd := &cobra.Command{
@@ -58,6 +60,28 @@ func Commands(con *console.SliverClient) []*cobra.Command {
flags.BindFlagCompletions(C2ProfileCmd, func(comp *carapace.ActionMap) {
(*comp)["name"] = generate.HTTPC2Completer(con)
})
generateC2ProfileCmd := &cobra.Command{
Use: consts.C2GenerateStr,
Short: "Generate a C2 Profile from a list of urls",
Long: help.GetHelpFor([]string{consts.C2ProfileStr + "." + consts.C2GenerateStr}),
Run: func(cmd *cobra.Command, args []string) {
GenerateC2ProfileCmd(cmd, con, args)
},
}
flags.Bind(consts.GenerateStr, false, generateC2ProfileCmd, func(f *pflag.FlagSet) {
f.StringP("file", "f", "", "Path to file containing URL list, /hello/there.txt one per line")
f.BoolP("import", "i", false, "Import the generated profile after creation")
f.StringP("name", "n", "", "HTTP C2 Profile name to save C2Profile as")
f.StringP("template", "t", consts.DefaultC2Profile, "HTTP C2 Profile to use as a template for the new profile")
})
flags.BindFlagCompletions(generateC2ProfileCmd, func(comp *carapace.ActionMap) {
(*comp)["template"] = generate.HTTPC2Completer(con)
})
C2ProfileCmd.AddCommand(generateC2ProfileCmd)
C2ProfileCmd.AddCommand(importC2ProfileCmd)
C2ProfileCmd.AddCommand(exportC2ProfileCmd)
+181
View File
@@ -0,0 +1,181 @@
package certificates
import (
"context"
"fmt"
"time"
"github.com/bishopfox/sliver/client/command/settings"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/jedib0t/go-pretty/v6/table"
"github.com/spf13/cobra"
"golang.org/x/term"
)
/*
Sliver Implant Framework
Copyright (C) 2024 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const (
timeFormat = "2006-01-02 15:04:05 UTC-0700"
)
// Defining the transport filters
const (
MTLSTransport uint32 = 1 << iota
HTTPSTransport
AllTransports
)
// Defining the role filters
const (
// Provide some separation between the options so that we do not have duplicate combinations
ServerRole uint32 = 8 << iota
ImplantRole
AllRoles
)
func CertificateInfoCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
// Since we are sending this value in a protobuf, we will give it a fixed bit size
// 32 is the smallest we can go
var chosenOptions uint32
if cmd.Flags().Changed("mtls") {
if cmd.Flags().Changed("https") {
chosenOptions = AllTransports
} else {
chosenOptions = MTLSTransport
}
} else if cmd.Flags().Changed("https") {
chosenOptions = HTTPSTransport
} else {
chosenOptions = AllTransports
}
if cmd.Flags().Changed("server") {
if cmd.Flags().Changed("implant") {
chosenOptions |= AllRoles
} else {
chosenOptions |= ServerRole
}
} else if cmd.Flags().Changed("implant") {
chosenOptions |= ImplantRole
} else {
chosenOptions |= AllRoles
}
request := &clientpb.CertificatesReq{
CategoryFilters: chosenOptions,
}
request.CN, _ = cmd.Flags().GetString("cn")
// Ask the server for information about certificates
certificateInfo, err := con.Rpc.GetCertificateInfo(context.Background(), request)
if err != nil {
con.PrintErrorf("could not get certificate information from database: %s", err.Error())
return
}
printCertificateInfo(con, certificateInfo.Info)
}
func checkCertExpiry(expiryTime time.Time) string {
if expiryTime.Before(time.Now()) || expiryTime.Equal(time.Now()) {
return console.Bold + console.Red
}
// One week is 168 hours - this is bad
if expiryTime.Before(time.Now().Add(168 * time.Hour)) {
return console.Bold + console.Red
}
// One month is approximately 730 hours - this is a warning
if expiryTime.Before(time.Now().Add(730 * time.Hour)) {
return console.Bold + console.Orange
}
return console.Normal
}
func printCertificateInfo(con *console.SliverClient, certData []*clientpb.CertificateData) {
// Get the terminal width
width, _, err := term.GetSize(0)
if err != nil {
width = 999
}
if len(certData) == 0 {
con.PrintWarnf("There are no certificates in the database matching the given parameters.\n")
return
}
tw := table.NewWriter()
tw.SetStyle(settings.GetTableStyle(con))
wideTermWidth := con.Settings.SmallTermWidth < width
if wideTermWidth {
tw.AppendHeader(table.Row{
"ID",
"Common Name",
"Creation Time",
"Certificate Type",
"Key Algorithm",
"Validity Start",
"Expires",
})
} else {
tw.AppendHeader(table.Row{
"ID",
"Common Name",
"Expires",
})
}
for _, cert := range certData {
rowColor := console.Normal
expiry, err := time.Parse(timeFormat, cert.ValidityExpiry)
// This should not error out, but if it does, the row will not be colored
if err == nil {
rowColor = checkCertExpiry(expiry)
}
if wideTermWidth {
tw.AppendRow(table.Row{
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.ID),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.CN),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.CreationTime),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.Type),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.KeyAlgorithm),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.ValidityStart),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.ValidityExpiry),
})
} else {
tw.AppendRow(table.Row{
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.ID),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.CN),
fmt.Sprintf(rowColor+"%s"+console.Normal, cert.ValidityExpiry),
})
}
}
tw.SortBy([]table.SortBy{{Name: "Expires", Mode: table.Dsc}})
con.Printf("%s\n", tw.Render())
}
+51
View File
@@ -0,0 +1,51 @@
package certificates
/*
Sliver Implant Framework
Copyright (C) 2024 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"github.com/bishopfox/sliver/client/command/flags"
"github.com/bishopfox/sliver/client/command/help"
"github.com/bishopfox/sliver/client/console"
consts "github.com/bishopfox/sliver/client/constants"
"github.com/spf13/cobra"
"github.com/spf13/pflag"
)
func Commands(con *console.SliverClient) []*cobra.Command {
certificatesCmd := &cobra.Command{
Use: consts.CertificatesStr,
Short: "Certificate management",
Long: help.GetHelpFor([]string{consts.CertificatesStr}),
Run: func(cmd *cobra.Command, args []string) {
CertificateInfoCmd(cmd, con, args)
},
GroupID: consts.GenericHelpGroup,
}
flags.Bind(consts.CertificatesStr, false, certificatesCmd, func(f *pflag.FlagSet) {
f.BoolP("mtls", "m", false, "Show MTLS certificates")
f.BoolP("https", "p", false, "Show HTTPS certificates")
f.BoolP("implant", "i", false, "Show implant certificates")
f.BoolP("server", "s", false, "Show server certificates")
f.StringP("cn", "c", "", "Show certificate information for a provided common name")
})
return []*cobra.Command{
certificatesCmd,
}
}
+141
View File
@@ -0,0 +1,141 @@
package clean
/*
Sliver Implant Framework
Copyright (C) 2023 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
import (
"context"
"github.com/AlecAivazis/survey/v2"
"github.com/bishopfox/sliver/client/command/flags"
"github.com/bishopfox/sliver/client/console"
"github.com/bishopfox/sliver/client/constants"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/protobuf/commonpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
"github.com/spf13/cobra"
)
// CleanCmd - Remove all profiles, beacons, sessions and implant builds (Builds and logs will still exist on disk in .sliver)
func CleanCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
con.Printf("This command will kill and remove all sessions, beacons and profiles \n")
confirm := false
prompt := &survey.Confirm{Message: "Are you sure you want to destroy everything?"}
survey.AskOne(prompt, &confirm)
if !confirm {
return
}
err := removeSessionsAndBeacons(con)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
err = removeImplantBuilds(con)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
err = removeProfiles(con)
if err != nil {
con.PrintErrorf("%s\n", err)
return
}
con.Printf("All done !\n")
}
func removeImplantBuilds(con *console.SliverClient) error {
builds, err := con.Rpc.ImplantBuilds(context.Background(), &commonpb.Empty{})
if err != nil {
return err
}
for name, _ := range builds.Configs {
_, err := con.Rpc.DeleteImplantBuild(context.Background(), &clientpb.DeleteReq{
Name: name,
})
if err != nil {
return err
}
}
return nil
}
func removeProfiles(con *console.SliverClient) error {
profiles, err := con.Rpc.ImplantProfiles(context.Background(), &commonpb.Empty{})
if err != nil {
return err
}
for _, profile := range profiles.Profiles {
_, err := con.Rpc.DeleteImplantProfile(context.Background(), &clientpb.DeleteReq{
Name: profile.Name,
})
if err != nil {
return err
}
}
return nil
}
func removeSessionsAndBeacons(con *console.SliverClient) error {
sessions, err := con.Rpc.GetSessions(context.Background(), &commonpb.Empty{})
if err != nil {
return err
}
for _, session := range sessions.Sessions {
_, err := con.Rpc.Kill(context.Background(), &sliverpb.KillReq{
Request: &commonpb.Request{
SessionID: session.ID,
Timeout: flags.DefaultTimeout,
},
Force: false,
})
if err != nil {
return err
}
}
beacons, err := con.Rpc.GetBeacons(context.Background(), &commonpb.Empty{})
if err != nil {
return err
}
for _, beacon := range beacons.Beacons {
_, err = con.Rpc.RmBeacon(context.Background(), &clientpb.Beacon{ID: beacon.ID})
if err != nil {
return err
}
}
return nil
}
// Commands returns the `exit` command.
func Command(con *console.SliverClient) []*cobra.Command {
return []*cobra.Command{{
Use: "clean",
Short: "Remove all profiles, beacons, sessions, implant builds and HTTP profiles (Builds and logs will still exist on disk in .sliver)",
Run: func(cmd *cobra.Command, args []string) {
CleanCmd(cmd, con, args)
},
GroupID: constants.GenericHelpGroup,
}}
}
+143 -48
View File
@@ -39,6 +39,7 @@ import (
"github.com/bishopfox/sliver/client/console"
consts "github.com/bishopfox/sliver/client/constants"
"github.com/bishopfox/sliver/client/core"
"github.com/bishopfox/sliver/client/packages"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
"github.com/bishopfox/sliver/util"
@@ -92,13 +93,15 @@ type ExtensionManifest struct {
}
type ExtCommand struct {
CommandName string `json:"command_name"`
Help string `json:"help"`
LongHelp string `json:"long_help"`
Files []*extensionFile `json:"files"`
Arguments []*extensionArgument `json:"arguments"`
Entrypoint string `json:"entrypoint"`
DependsOn string `json:"depends_on"`
CommandName string `json:"command_name"`
Help string `json:"help"`
LongHelp string `json:"long_help"`
Files []*extensionFile `json:"files"`
Arguments []*extensionArgument `json:"arguments"`
Entrypoint string `json:"entrypoint"`
DependsOn string `json:"depends_on"`
Init string `json:"init"`
Schema *packages.OutputSchema `json:"schema"`
Manifest *ExtensionManifest
}
@@ -200,6 +203,7 @@ func convertOldManifest(old *ExtensionManifest_) *ExtensionManifest {
Entrypoint: old.Entrypoint,
Files: old.Files,
Arguments: old.Arguments,
Schema: nil,
},
},
}
@@ -227,9 +231,13 @@ func ParseExtensionManifest(data []byte) (*ExtensionManifest, error) {
//yes, ok, lets jigger it to a new manifest
extManifest = convertOldManifest(oldmanifest)
}
//pass ref to manifest to each command
//pass ref to manifest to each command and initialize output schema if applicable
for i := range extManifest.ExtCommand {
extManifest.ExtCommand[i].Manifest = extManifest
command := extManifest.ExtCommand[i]
command.Manifest = extManifest
if command.Schema != nil {
command.Schema.IngestColumns()
}
}
return extManifest, validManifest(extManifest)
}
@@ -262,6 +270,11 @@ func validManifest(manifest *ExtensionManifest) error {
if extManifest.Help == "" {
return errors.New("missing `help` field in extension manifest")
}
if extManifest.Schema != nil {
if !packages.IsValidSchemaType(extManifest.Schema.Name) {
return fmt.Errorf("%s is not a valid schema type", extManifest.Schema.Name)
}
}
}
return nil
}
@@ -371,38 +384,50 @@ func loadExtension(goos string, goarch string, checkCache bool, ext *ExtCommand,
}
extensionList = extList.Names
}
depLoaded := false
//extensionList contains all *implant* loaded extensions. Is a sha256 sum of the relevant file.
//get the file hash to compare against implant extensions later
toberunfilepath, err := ext.getFileForTarget(goos, goarch)
if err != nil {
return err
}
//we need to check the dependent if it's a bof
if ext.DependsOn != "" {
//verify extension is in loaded list
if extension, found := loadedExtensions[ext.DependsOn]; found {
toberunfilepath, err = extension.getFileForTarget(goos, goarch)
if err != nil {
return err
}
} else {
// handle error
return fmt.Errorf("attempted to load non-existing extension: %s", ext.DependsOn)
}
}
//todo, maybe cache these values somewhere
toberunfiledata, err := os.ReadFile(toberunfilepath)
if err != nil {
return err
}
if len(toberunfiledata) == 0 {
//read an empty file, bail out
return errors.New("read empty extension file content")
}
bd := sha256.Sum256(toberunfiledata)
toberunhash := hex.EncodeToString(bd[:])
for _, extName := range extensionList {
if !depLoaded {
if extN, ok := loadedExtensions[cmd.Name()]; ok && extName == extN.DependsOn {
depLoaded = true
}
}
//check if the file suffixes are the same (should only be relevant to the same extension package), and don't load the file if it's already there
extN := loadedExtensions[cmd.Name()]
if extN != nil {
//confirm we haven't already loaded the file
for _, extf := range extN.Files {
if strings.HasSuffix(binPath, extf.Path) {
//file already exists, no need to load it
return nil
}
}
}
if extN, ok := loadedExtensions[cmd.Name()]; ok && extN.CommandName == extName {
//check if extension we are trying to run (ext.CommandName or ext.DependsOn, for bofs) is already loaded
if extName == toberunhash {
//exists on the other side, exit early
return nil
}
}
// Extension not found, let's load it
if filepath.Ext(binPath) == ".o" {
// BOFs are not loaded by the DLL loader, but we make sure the loader itself is loaded
// Auto load the coff loader if we have it
if !depLoaded {
if errLoad := loadDep(goos, goarch, ext.DependsOn, cmd, con); errLoad != nil {
return errLoad
}
}
return nil
// BOFs are not loaded by the DLL loader, but we need to load the loader (more load more good)
if ext.DependsOn != "" {
return loadDep(goos, goarch, ext.DependsOn, cmd, con)
}
binData, err := os.ReadFile(binPath)
if err != nil {
@@ -414,17 +439,45 @@ func loadExtension(goos string, goarch string, checkCache bool, ext *ExtCommand,
return nil
}
func registerExtension(goos string, _ *ExtCommand, binData []byte, cmd *cobra.Command, con *console.SliverClient) error {
func registerExtension(goos string, ext *ExtCommand, binData []byte, cmd *cobra.Command, con *console.SliverClient) error {
//set extension name to a hash of the data to avoid loading more than one instance
bd := sha256.Sum256(binData)
name := hex.EncodeToString(bd[:])
sess, beac := con.ActiveTarget.GetInteractive()
ctrl := make(chan bool)
//first time run of an extension will require some waiting depending on the size
if sess != nil {
msg := fmt.Sprintf("Sending %s to implant ...", ext.CommandName)
con.SpinUntil(msg, ctrl)
}
//don't block if we are in beacon mode
if beac != nil && sess == nil {
go func() {
registerResp, err := con.Rpc.RegisterExtension(context.Background(), &sliverpb.RegisterExtensionReq{
Name: name,
Data: binData,
OS: goos,
Init: ext.Init,
Request: con.ActiveTarget.Request(cmd),
})
if err != nil {
con.PrintErrorf("Error registering extension: %s\n", err)
}
if registerResp.Response != nil && registerResp.Response.Err != "" {
con.PrintErrorf("Error registering extension: %s\n", errors.New(registerResp.Response.Err))
}
}()
return nil
}
//session mode (hopefully)
registerResp, err := con.Rpc.RegisterExtension(context.Background(), &sliverpb.RegisterExtensionReq{
Name: name,
Data: binData,
OS: goos,
//Init: ext.Init, ?
Name: name,
Data: binData,
OS: goos,
Request: con.ActiveTarget.Request(cmd),
})
ctrl <- true
<-ctrl
if err != nil {
return err
}
@@ -525,6 +578,33 @@ func runExtensionCmd(cmd *cobra.Command, con *console.SliverClient, args []strin
}
bd := sha256.Sum256(extdata)
name := hex.EncodeToString(bd[:])
if isBOF {
//if we are using a bof, we are actually calling the coffloader extension - so get the file from dep ref and use that shasum
if extension, found := loadedExtensions[ext.DependsOn]; found {
dep, err := extension.getFileForTarget(goos, goarch)
if err != nil {
con.PrintErrorf("could not get file for extension %s", ext.DependsOn)
return
}
depdata, err := os.ReadFile(dep)
if err != nil {
con.PrintErrorf("dep read file error: %s\n", err)
return
}
if len(depdata) == 0 {
con.PrintErrorf("read empty file: %s\n", dep)
return
}
bd = sha256.Sum256(depdata)
name = hex.EncodeToString(bd[:])
} else {
// handle error
con.PrintErrorf("attempted to load non-existing extension: %s", ext.DependsOn)
return
}
}
callExtResp, err := con.Rpc.CallExtension(context.Background(), &sliverpb.CallExtensionReq{
Name: name,
Export: entryPoint,
@@ -545,23 +625,38 @@ func runExtensionCmd(cmd *cobra.Command, con *console.SliverClient, args []strin
con.PrintErrorf("Failed to decode call ext response %s\n", err)
return
}
PrintExtOutput(extName, ext.CommandName, callExtResp, con)
PrintExtOutput(extName, ext.CommandName, ext.Schema, callExtResp, con)
})
con.PrintAsyncResponse(callExtResp.Response)
} else {
PrintExtOutput(extName, ext.CommandName, callExtResp, con)
PrintExtOutput(extName, ext.CommandName, ext.Schema, callExtResp, con)
}
}
// PrintExtOutput - Print the ext execution output.
func PrintExtOutput(extName string, commandName string, callExtension *sliverpb.CallExtension, con *console.SliverClient) {
func PrintExtOutput(extName string, commandName string, outputSchema *packages.OutputSchema, callExtension *sliverpb.CallExtension, con *console.SliverClient) {
if extName == commandName {
con.PrintInfof("Successfully executed %s", extName)
con.PrintInfof("Successfully executed %s\n", extName)
} else {
con.PrintInfof("Successfully executed %s (%s)", commandName, extName)
con.PrintInfof("Successfully executed %s (%s)\n", commandName, extName)
}
if 0 < len(string(callExtension.Output)) {
con.PrintInfof("Got output:\n%s", callExtension.Output)
if outputSchema == nil {
con.PrintInfof("Got output:\n%s", callExtension.Output)
} else {
// Get output schema
schema := packages.GetNewPackageOutput(outputSchema.Name)
if schema != nil {
ingestErr := schema.IngestData(callExtension.Output, outputSchema.Columns(), outputSchema.GroupBy)
if ingestErr != nil {
con.PrintInfof("Got output:\n%s", callExtension.Output)
} else {
con.Printf("%s\n", schema.CreateTable())
}
} else {
con.PrintInfof("Got output:\n%s", callExtension.Output)
}
}
}
if callExtension.Response != nil && callExtension.Response.Err != "" {
con.PrintErrorf(callExtension.Response.Err)
@@ -579,7 +674,7 @@ func getExtArgs(_ *cobra.Command, args []string, _ string, ext *ExtCommand) ([]b
missingRequiredArgs := make([]string, 0)
// If we have an extension that expects a single string, but more than one has been parsed, combine them
if len(ext.Arguments) == 1 && strings.Contains(ext.Arguments[0].Type, "string") {
if len(ext.Arguments) == 1 && strings.Contains(ext.Arguments[0].Type, "string") && len(args) > 0 {
// The loop below will only read the first element of args because ext.Arguments is 1
args[0] = strings.Join(args, " ")
}
+22
View File
@@ -184,6 +184,28 @@ func Commands(con *console.SliverClient) []*cobra.Command {
}
profilesCmd.AddCommand(profilesNewCmd)
profilesStageCmd := &cobra.Command{
Use: consts.StageStr,
Short: "Generate implant from a profile and encode or encrypt it",
Args: cobra.ExactArgs(1),
Long: help.GetHelpFor([]string{consts.ProfilesStr, consts.StageStr}),
Run: func(cmd *cobra.Command, args []string) {
ProfilesStageCmd(cmd, con, args)
},
}
flags.Bind("profiles", false, profilesStageCmd, func(f *pflag.FlagSet) {
f.StringP("save", "s", "", "directory/file to the binary to")
f.StringP("name", "n", "", "Implant name")
f.StringP("aes-encrypt-key", "k", "", "AES Encryption Key")
f.StringP("aes-encrypt-iv", "i", "", "AES Encryption IV")
f.StringP("rc4-encrypt-key", "r", "", "RC4 encryption key")
f.BoolP("prepend-size", "p", false, "Prepend stage size")
f.StringP("compress", "c", "", "Compress stage (zlib, gzip, deflate9 or deflate)")
})
carapace.Gen(profilesStageCmd).PositionalCompletion(ProfileNameCompleter(con))
profilesCmd.AddCommand(profilesStageCmd)
// Session flags and completions.
coreImplantFlags("session", profilesNewCmd)
compileImplantFlags("session", profilesNewCmd)
+9
View File
@@ -108,6 +108,7 @@ var (
// Profiles
consts.ProfilesStr + sep + consts.NewStr: newProfileHelp,
consts.ProfilesStr + sep + consts.GenerateStr: generateProfileHelp,
consts.ProfilesStr + sep + consts.StageStr: generateProfileStageHelp,
// Reactions
consts.ReactionStr: reactionHelp,
@@ -121,6 +122,7 @@ var (
// HTTP C2
consts.C2ProfileStr: c2ProfilesHelp,
consts.C2ProfileStr + sep + consts.C2GenerateStr: c2GenerateHelp,
}
jobsHelp = `[[.Bold]]Command:[[.Normal]] jobs <options>
@@ -249,6 +251,9 @@ settings. Generated implants will still have per-binary certificates/obfuscation
generateProfileHelp = `[[.Bold]]Command:[[.Normal]] generate [name] <options>
[[.Bold]]About:[[.Normal]] Generate an implant from a saved profile (see 'profiles new --help').`
generateProfileStageHelp = `[[.Bold]]Command:[[.Normal]] stage [name] <options>
[[.Bold]]About:[[.Normal]] Generate and encrypt or encode an implant from a saved profile (see 'profiles stage --help').`
msfHelp = `[[.Bold]]Command:[[.Normal]] msf [--lhost] <options>
[[.Bold]]About:[[.Normal]] Execute a metasploit payload in the current process.`
@@ -1292,6 +1297,10 @@ Sliver uses the same hash identifiers as Hashcat (use the #):
C2ProfileImportStr = `[[.Bold]]Command:[[.Normal]] Import
[[.Bold]]About:[[.Normal]] Load custom HTTP C2 profiles.
`
c2GenerateHelp = `[[.Bold]]Command:[[.Normal]] C2 Profile generate
[[.Bold]]About:[[.Normal]] Generate C2 profile using a file containing urls.
Optionaly import profile or use another profile as a base template for the new profile.
`
grepHelp = `[[.Bold]]Command:[[.Normal]] grep [flags / options] <search pattern> <path>
[[.Bold]]About:[[.Normal]] Search a file or path for a search pattern
+4
View File
@@ -26,6 +26,8 @@ import (
"github.com/bishopfox/sliver/client/command/beacons"
"github.com/bishopfox/sliver/client/command/builders"
"github.com/bishopfox/sliver/client/command/c2profiles"
"github.com/bishopfox/sliver/client/command/certificates"
"github.com/bishopfox/sliver/client/command/clean"
"github.com/bishopfox/sliver/client/command/crack"
"github.com/bishopfox/sliver/client/command/creds"
"github.com/bishopfox/sliver/client/command/exit"
@@ -93,6 +95,8 @@ func ServerCommands(con *client.SliverClient, serverCmds func() []*cobra.Command
operators.Commands,
creds.Commands,
crack.Commands,
certificates.Commands,
clean.Command,
)
// C2 Network
+1 -1
View File
@@ -227,7 +227,7 @@ func renderTaskResponse(task *clientpb.BeaconTask, con *console.SliverClient) {
con.PrintErrorf("Failed to decode task response: %s\n", err)
return
}
extensions.PrintExtOutput("", "", callExtension, con)
extensions.PrintExtOutput("", "", nil, callExtension, con)
// ---------------------
// Exec commands
+2
View File
@@ -264,11 +264,13 @@ func downloadAsset(client *http.Client, asset *version.Asset, saveTo string) err
if err != nil {
return err
}
defer writer.Close()
resp, err := client.Get(asset.BrowserDownloadURL)
if err != nil {
return err
}
defer resp.Body.Close()
bar := pb.Full.Start64(limit)
barReader := bar.NewProxyReader(resp.Body)
+1 -1
View File
@@ -88,7 +88,7 @@ func WGConfigCmd(cmd *cobra.Command, con *console.SliverClient, args []string) {
save, _ := cmd.Flags().GetString("save")
if save == "" {
con.PrintInfof("New client config:")
con.PrintInfof("New client config:\n")
con.Println(output.String())
} else {
if !strings.HasSuffix(save, ".conf") {
+2
View File
@@ -133,6 +133,7 @@ const (
TasksStr = "tasks"
CancelStr = "cancel"
GenerateStr = "generate"
C2GenerateStr = "generate"
RegenerateStr = "regenerate"
CompilerInfoStr = "info"
MsfStagerStr = "msf-stager"
@@ -146,6 +147,7 @@ const (
C2ProfileStr = "c2profiles"
ImportC2ProfileStr = "import"
ExportC2ProfileStr = "export"
CertificatesStr = "certificates"
// Generic.
+254
View File
@@ -0,0 +1,254 @@
package packages
import (
"encoding/json"
"fmt"
"slices"
"strconv"
"strings"
"github.com/jedib0t/go-pretty/v6/table"
)
/*
Sliver Implant Framework
Copyright (C) 2024 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const (
KeyValueSchemaName = "key-value"
GroupedKeyValueSchemaName = "grouped-key-value"
)
var validSchemas []string = []string{KeyValueSchemaName, GroupedKeyValueSchemaName}
type OutputSchema struct {
Name string `json:"name"`
RawColumns string `json:"columns"`
GroupBy string `json:"group_by"`
cols []string `json:"-"`
}
func (schema *OutputSchema) IngestColumns() {
schema.cols = strings.Split(schema.RawColumns, ",")
}
func (schema *OutputSchema) Columns() []string {
return schema.cols
}
type PackageOutput interface {
// The names of columns followed by any other strings needed, like the data itself
// Each type that implements this interface will have to make sure the correct number
// of strings are provided
IngestData([]byte, []string, ...string) error
// Renders the data into a table
CreateTable() string
Name() string
}
func IsValidSchemaType(name string) bool {
return slices.Contains(validSchemas, name)
}
func GetNewPackageOutput(name string) PackageOutput {
switch name {
case KeyValueSchemaName:
return &KeyValuePairs{}
case GroupedKeyValueSchemaName:
return &GroupedKeyValuePairs{}
default:
return nil
}
}
/*
This output schema is suitable for data that is an array of key-value pairs
*/
type KeyValueData map[string]string
type KeyValuePairs struct {
columnOrder []string
data []KeyValueData
// This field is optional
columnToGroupBy string
}
/*
This output schema is suitable for one or more key-values pairs grouped
by a group name. This schema assumes the data is already grouped and
does not perform the grouping itself.
*/
type GroupedKeyValuePairs struct {
groupedBy string
columnOrder []string
data map[string][]KeyValueData
}
func (kvd *KeyValueData) UnmarshalJSON(data []byte) error {
var raw map[string]interface{}
if err := json.Unmarshal(data, &raw); err != nil {
return err
}
*kvd = make(KeyValueData, len(raw))
for key, value := range raw {
key = strings.ToLower(key)
switch v := value.(type) {
case float64:
(*kvd)[key] = strconv.FormatFloat(v, 'f', -1, 64)
case string:
// Newlines and extra whitespace mess up the table
(*kvd)[key] = strings.TrimSpace(v)
case bool:
(*kvd)[key] = strconv.FormatBool(v)
default:
return fmt.Errorf("unsupported type %T", v)
}
}
return nil
}
func (kvp *KeyValuePairs) IngestData(data []byte, order []string, arguments ...string) error {
if len(arguments) > 1 {
// Only an optional a column to group by
return fmt.Errorf("unexpected number of arguments - expected only an optional column name to group by")
}
groupByColumn := ""
if len(arguments) == 1 {
groupByColumn = arguments[0]
}
err := json.Unmarshal(data, &kvp.data)
if err != nil {
return err
}
if len(kvp.data) == 0 {
return nil
}
// If the wrong columns were provided, there will be empty cells
kvp.columnOrder = order
// If an invalid column name is supplied, it will have no effect
kvp.columnToGroupBy = groupByColumn
return nil
}
func (kvp *KeyValuePairs) CreateTable() string {
if len(kvp.data) == 0 {
return ""
}
t := table.NewWriter()
header := table.Row{}
for _, column := range kvp.columnOrder {
header = append(header, column)
}
t.AppendHeader(header)
for _, dataRow := range kvp.data {
tableRow := table.Row{}
for _, column := range kvp.columnOrder {
column = strings.ToLower(column)
tableRow = append(tableRow, dataRow[column])
}
t.AppendRow(tableRow)
}
t.SetColumnConfigs([]table.ColumnConfig{{Name: kvp.columnToGroupBy, AutoMerge: true}})
if kvp.columnToGroupBy != "" {
t.SortBy([]table.SortBy{{Name: kvp.columnToGroupBy, Mode: table.Asc}})
} else {
t.SortBy([]table.SortBy{{Number: 1, Mode: table.Asc}})
}
return t.Render()
}
func (kvp *KeyValuePairs) Name() string {
return KeyValueSchemaName
}
func (gkvp *GroupedKeyValuePairs) IngestData(data []byte, order []string, arguments ...string) error {
if len(arguments) != 1 {
return fmt.Errorf("unexpected number of arguments - a grouped by column name")
}
groupedBy := arguments[0]
err := json.Unmarshal(data, &gkvp.data)
if err != nil {
return err
}
if len(gkvp.data) == 0 {
return nil
}
// If the wrong columns were provided, there will be empty cells
gkvp.columnOrder = order
// If an invalid grouped by column is provided, the table will not render correctly,
// but there will not be any panics
gkvp.groupedBy = groupedBy
return nil
}
func (gkvp *GroupedKeyValuePairs) CreateTable() string {
if len(gkvp.data) == 0 {
return ""
}
t := table.NewWriter()
header := table.Row{}
for _, column := range gkvp.columnOrder {
header = append(header, column)
}
t.AppendHeader(header)
groupColumn := strings.ToLower(gkvp.groupedBy)
for groupName, groupData := range gkvp.data {
for _, group := range groupData {
tableRow := table.Row{}
for _, column := range gkvp.columnOrder {
column = strings.ToLower(column)
if column == groupColumn {
tableRow = append(tableRow, groupName)
} else {
tableRow = append(tableRow, group[column])
}
}
t.AppendRow(tableRow)
}
}
t.SetColumnConfigs([]table.ColumnConfig{{Name: gkvp.groupedBy, AutoMerge: true}})
if gkvp.groupedBy != "" {
t.SortBy([]table.SortBy{{Name: gkvp.groupedBy, Mode: table.Asc}})
} else {
t.SortBy([]table.SortBy{{Number: 1, Mode: table.Asc}})
}
return t.Render()
}
func (gkvp *GroupedKeyValuePairs) Name() string {
return GroupedKeyValueSchemaName
}
+6
View File
@@ -7,6 +7,7 @@ import { useRouter } from "next/router";
import Markdown from "react-markdown";
import remarkGfm from "remark-gfm";
import AsciinemaPlayer from "./asciinema";
import Youtube from "./youtube";
export type MarkdownProps = {
key?: string;
@@ -131,6 +132,11 @@ const MarkdownViewer = (props: MarkdownProps) => {
);
}
if (lang === "youtube") {
const embedId = (children as string) || "";
return <Youtube embedId={embedId.trim()} />;
}
if (lang === "asciinema") {
const asciiCast: MarkdownAsciiCast = JSON.parse(
children as string
+11 -12
View File
@@ -42,7 +42,7 @@ export default function TutorialCard(props: TutorialCardCardProps) {
</div>
<CardFooter className="absolute bg-black/40 bottom-0 z-100 border-t-1 border-default-600 dark:border-default-100">
<div className="flex w-full items-center">
<div className="flex flex-row w-full items-center">
<p
className={
props.italicDescription
@@ -54,17 +54,16 @@ export default function TutorialCard(props: TutorialCardCardProps) {
</p>
{props.showButton ? (
<div className="justify-items-end">
<Button
variant="ghost"
color="warning"
size="sm"
onPress={props.onPress}
>
{props.buttonText ? props.buttonText : "Read Tutorial"}{" "}
<FontAwesomeIcon icon={faChevronRight} />
</Button>
</div>
<Button
variant="ghost"
color="warning"
size="sm"
onPress={props.onPress}
className="ml-auto"
>
{props.buttonText ? props.buttonText : "Read Tutorial"}{" "}
<FontAwesomeIcon icon={faChevronRight} />
</Button>
) : (
<></>
)}
+22
View File
@@ -0,0 +1,22 @@
export type YoutubeProps = {
className?: string;
width?: number;
height?: number;
embedId: string;
};
export default function Youtube(props: YoutubeProps) {
return (
<div className={props.className}>
<iframe
width={`${props.width ? props.width : 640}`}
height={`${props.height ? props.height : 360}`}
src={`https://youtu.be/embed/${props.embedId}`}
allow="accelerometer; autoplay; clipboard-write; encrypted-media; gyroscope; picture-in-picture"
allowFullScreen
/>
</div>
);
}
+63 -71
View File
@@ -19,7 +19,7 @@
"framer-motion": "^10.16.16",
"fuse.js": "^7.0.0",
"lunr": "^2.3.9",
"next": "14.0.4",
"next": "^14.2.3",
"next-themes": "^0.2.1",
"react": "18.2.0",
"react-dom": "18.2.0",
@@ -383,9 +383,9 @@
}
},
"node_modules/@next/env": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/env/-/env-14.0.4.tgz",
"integrity": "sha512-irQnbMLbUNQpP1wcE5NstJtbuA/69kRfzBrpAD7Gsn8zm/CY6YQYc3HQBz8QPxwISG26tIm5afvvVbu508oBeQ=="
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/env/-/env-14.2.3.tgz",
"integrity": "sha512-W7fd7IbkfmeeY2gXrzJYDx8D2lWKbVoTIj1o1ScPHNzvp30s1AuoEFSdr39bC5sjxJaxTtq3OTCZboNp0lNWHA=="
},
"node_modules/@next/eslint-plugin-next": {
"version": "14.0.4",
@@ -397,9 +397,9 @@
}
},
"node_modules/@next/swc-darwin-arm64": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.0.4.tgz",
"integrity": "sha512-mF05E/5uPthWzyYDyptcwHptucf/jj09i2SXBPwNzbgBNc+XnwzrL0U6BmPjQeOL+FiB+iG1gwBeq7mlDjSRPg==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-arm64/-/swc-darwin-arm64-14.2.3.tgz",
"integrity": "sha512-3pEYo/RaGqPP0YzwnlmPN2puaF2WMLM3apt5jLW2fFdXD9+pqcoTzRk+iZsf8ta7+quAe4Q6Ms0nR0SFGFdS1A==",
"cpu": [
"arm64"
],
@@ -412,9 +412,9 @@
}
},
"node_modules/@next/swc-darwin-x64": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.0.4.tgz",
"integrity": "sha512-IZQ3C7Bx0k2rYtrZZxKKiusMTM9WWcK5ajyhOZkYYTCc8xytmwSzR1skU7qLgVT/EY9xtXDG0WhY6fyujnI3rw==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-darwin-x64/-/swc-darwin-x64-14.2.3.tgz",
"integrity": "sha512-6adp7waE6P1TYFSXpY366xwsOnEXM+y1kgRpjSRVI2CBDOcbRjsJ67Z6EgKIqWIue52d2q/Mx8g9MszARj8IEA==",
"cpu": [
"x64"
],
@@ -427,9 +427,9 @@
}
},
"node_modules/@next/swc-linux-arm64-gnu": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.0.4.tgz",
"integrity": "sha512-VwwZKrBQo/MGb1VOrxJ6LrKvbpo7UbROuyMRvQKTFKhNaXjUmKTu7wxVkIuCARAfiI8JpaWAnKR+D6tzpCcM4w==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-gnu/-/swc-linux-arm64-gnu-14.2.3.tgz",
"integrity": "sha512-cuzCE/1G0ZSnTAHJPUT1rPgQx1w5tzSX7POXSLaS7w2nIUJUD+e25QoXD/hMfxbsT9rslEXugWypJMILBj/QsA==",
"cpu": [
"arm64"
],
@@ -442,9 +442,9 @@
}
},
"node_modules/@next/swc-linux-arm64-musl": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.0.4.tgz",
"integrity": "sha512-8QftwPEW37XxXoAwsn+nXlodKWHfpMaSvt81W43Wh8dv0gkheD+30ezWMcFGHLI71KiWmHK5PSQbTQGUiidvLQ==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-linux-arm64-musl/-/swc-linux-arm64-musl-14.2.3.tgz",
"integrity": "sha512-0D4/oMM2Y9Ta3nGuCcQN8jjJjmDPYpHX9OJzqk42NZGJocU2MqhBq5tWkJrUQOQY9N+In9xOdymzapM09GeiZw==",
"cpu": [
"arm64"
],
@@ -457,9 +457,9 @@
}
},
"node_modules/@next/swc-linux-x64-gnu": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.0.4.tgz",
"integrity": "sha512-/s/Pme3VKfZAfISlYVq2hzFS8AcAIOTnoKupc/j4WlvF6GQ0VouS2Q2KEgPuO1eMBwakWPB1aYFIA4VNVh667A==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-gnu/-/swc-linux-x64-gnu-14.2.3.tgz",
"integrity": "sha512-ENPiNnBNDInBLyUU5ii8PMQh+4XLr4pG51tOp6aJ9xqFQ2iRI6IH0Ds2yJkAzNV1CfyagcyzPfROMViS2wOZ9w==",
"cpu": [
"x64"
],
@@ -472,9 +472,9 @@
}
},
"node_modules/@next/swc-linux-x64-musl": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.0.4.tgz",
"integrity": "sha512-m8z/6Fyal4L9Bnlxde5g2Mfa1Z7dasMQyhEhskDATpqr+Y0mjOBZcXQ7G5U+vgL22cI4T7MfvgtrM2jdopqWaw==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-linux-x64-musl/-/swc-linux-x64-musl-14.2.3.tgz",
"integrity": "sha512-BTAbq0LnCbF5MtoM7I/9UeUu/8ZBY0i8SFjUMCbPDOLv+un67e2JgyN4pmgfXBwy/I+RHu8q+k+MCkDN6P9ViQ==",
"cpu": [
"x64"
],
@@ -487,9 +487,9 @@
}
},
"node_modules/@next/swc-win32-arm64-msvc": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.0.4.tgz",
"integrity": "sha512-7Wv4PRiWIAWbm5XrGz3D8HUkCVDMMz9igffZG4NB1p4u1KoItwx9qjATHz88kwCEal/HXmbShucaslXCQXUM5w==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-win32-arm64-msvc/-/swc-win32-arm64-msvc-14.2.3.tgz",
"integrity": "sha512-AEHIw/dhAMLNFJFJIJIyOFDzrzI5bAjI9J26gbO5xhAKHYTZ9Or04BesFPXiAYXDNdrwTP2dQceYA4dL1geu8A==",
"cpu": [
"arm64"
],
@@ -502,9 +502,9 @@
}
},
"node_modules/@next/swc-win32-ia32-msvc": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.0.4.tgz",
"integrity": "sha512-zLeNEAPULsl0phfGb4kdzF/cAVIfaC7hY+kt0/d+y9mzcZHsMS3hAS829WbJ31DkSlVKQeHEjZHIdhN+Pg7Gyg==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-win32-ia32-msvc/-/swc-win32-ia32-msvc-14.2.3.tgz",
"integrity": "sha512-vga40n1q6aYb0CLrM+eEmisfKCR45ixQYXuBXxOOmmoV8sYST9k7E3US32FsY+CkkF7NtzdcebiFT4CHuMSyZw==",
"cpu": [
"ia32"
],
@@ -517,9 +517,9 @@
}
},
"node_modules/@next/swc-win32-x64-msvc": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.0.4.tgz",
"integrity": "sha512-yEh2+R8qDlDCjxVpzOTEpBLQTEFAcP2A8fUFLaWNap9GitYKkKv1//y2S6XY6zsR4rCOPRpU7plYDR+az2n30A==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/@next/swc-win32-x64-msvc/-/swc-win32-x64-msvc-14.2.3.tgz",
"integrity": "sha512-Q1/zm43RWynxrO7lW4ehciQVj+5ePBhOK+/K2P7pLFX3JaJ/IZVC69SHidrmZSOkqz7ECIOhhy7XhAFG4JYyHA==",
"cpu": [
"x64"
],
@@ -2815,11 +2815,17 @@
"integrity": "sha512-2/U3GXA6YiPYQDLGwtGlnNgKYBSwCFIHf8Y9LUY5VATHdtbLlU0Y1R3QoBnT0aB4qv/BEiVVsj7LJXoQCgJ2vA==",
"dev": true
},
"node_modules/@swc/counter": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@swc/counter/-/counter-0.1.3.tgz",
"integrity": "sha512-e2BR4lsJkkRlKZ/qCHPw9ZaSxc0MVUd7gtbtaB7aMvHeJVYe8sOB8DBZkP2DtISHGSku9sCK6T6cnY0CtXrOCQ=="
},
"node_modules/@swc/helpers": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.2.tgz",
"integrity": "sha512-E4KcWTpoLHqwPHLxidpOqQbcrZVgi0rsmmZXUle1jXmJfuIf/UWpczUJ7MZZ5tlxytgJXyp0w4PGkkeLiuIdZw==",
"version": "0.5.5",
"resolved": "https://registry.npmjs.org/@swc/helpers/-/helpers-0.5.5.tgz",
"integrity": "sha512-KGYxvIOXcceOAbEk4bi/dVLEK9z8sZ0uBB3Il5b1rhfClSpcX0yfRO0KmTkqR2cnQDymwLB+25ZyMzICg/cm/A==",
"dependencies": {
"@swc/counter": "^0.1.3",
"tslib": "^2.4.0"
}
},
@@ -3515,9 +3521,9 @@
}
},
"node_modules/caniuse-lite": {
"version": "1.0.30001570",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001570.tgz",
"integrity": "sha512-+3e0ASu4sw1SWaoCtvPeyXp+5PsjigkSt8OXZbF9StH5pQWbxEjLAZE3n8Aup5udop1uRiKA7a4utUk/uoSpUw==",
"version": "1.0.30001614",
"resolved": "https://registry.npmjs.org/caniuse-lite/-/caniuse-lite-1.0.30001614.tgz",
"integrity": "sha512-jmZQ1VpmlRwHgdP1/uiKzgiAuGOfLEJsYFP4+GBou/QQ4U6IOJCB4NP1c+1p9RGLpwObcT94jA5/uO+F1vBbog==",
"funding": [
{
"type": "opencollective",
@@ -4773,11 +4779,6 @@
"node": ">=10.13.0"
}
},
"node_modules/glob-to-regexp": {
"version": "0.4.1",
"resolved": "https://registry.npmjs.org/glob-to-regexp/-/glob-to-regexp-0.4.1.tgz",
"integrity": "sha512-lkX1HJXwyMcprw/5YUZc2s7DrpAiHB21/V+E1rHUrVNokkvB6bqMzT0VfV6/86ZNabt1k14YOIaT7nDvOX3Iiw=="
},
"node_modules/globals": {
"version": "13.24.0",
"resolved": "https://registry.npmjs.org/globals/-/globals-13.24.0.tgz",
@@ -6587,18 +6588,17 @@
"dev": true
},
"node_modules/next": {
"version": "14.0.4",
"resolved": "https://registry.npmjs.org/next/-/next-14.0.4.tgz",
"integrity": "sha512-qbwypnM7327SadwFtxXnQdGiKpkuhaRLE2uq62/nRul9cj9KhQ5LhHmlziTNqUidZotw/Q1I9OjirBROdUJNgA==",
"version": "14.2.3",
"resolved": "https://registry.npmjs.org/next/-/next-14.2.3.tgz",
"integrity": "sha512-dowFkFTR8v79NPJO4QsBUtxv0g9BrS/phluVpMAt2ku7H+cbcBJlopXjkWlwxrk/xGqMemr7JkGPGemPrLLX7A==",
"dependencies": {
"@next/env": "14.0.4",
"@swc/helpers": "0.5.2",
"@next/env": "14.2.3",
"@swc/helpers": "0.5.5",
"busboy": "1.6.0",
"caniuse-lite": "^1.0.30001406",
"caniuse-lite": "^1.0.30001579",
"graceful-fs": "^4.2.11",
"postcss": "8.4.31",
"styled-jsx": "5.1.1",
"watchpack": "2.4.0"
"styled-jsx": "5.1.1"
},
"bin": {
"next": "dist/bin/next"
@@ -6607,18 +6607,19 @@
"node": ">=18.17.0"
},
"optionalDependencies": {
"@next/swc-darwin-arm64": "14.0.4",
"@next/swc-darwin-x64": "14.0.4",
"@next/swc-linux-arm64-gnu": "14.0.4",
"@next/swc-linux-arm64-musl": "14.0.4",
"@next/swc-linux-x64-gnu": "14.0.4",
"@next/swc-linux-x64-musl": "14.0.4",
"@next/swc-win32-arm64-msvc": "14.0.4",
"@next/swc-win32-ia32-msvc": "14.0.4",
"@next/swc-win32-x64-msvc": "14.0.4"
"@next/swc-darwin-arm64": "14.2.3",
"@next/swc-darwin-x64": "14.2.3",
"@next/swc-linux-arm64-gnu": "14.2.3",
"@next/swc-linux-arm64-musl": "14.2.3",
"@next/swc-linux-x64-gnu": "14.2.3",
"@next/swc-linux-x64-musl": "14.2.3",
"@next/swc-win32-arm64-msvc": "14.2.3",
"@next/swc-win32-ia32-msvc": "14.2.3",
"@next/swc-win32-x64-msvc": "14.2.3"
},
"peerDependencies": {
"@opentelemetry/api": "^1.1.0",
"@playwright/test": "^1.41.2",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"sass": "^1.3.0"
@@ -6627,6 +6628,9 @@
"@opentelemetry/api": {
"optional": true
},
"@playwright/test": {
"optional": true
},
"sass": {
"optional": true
}
@@ -8431,18 +8435,6 @@
"url": "https://opencollective.com/unified"
}
},
"node_modules/watchpack": {
"version": "2.4.0",
"resolved": "https://registry.npmjs.org/watchpack/-/watchpack-2.4.0.tgz",
"integrity": "sha512-Lcvm7MGST/4fup+ifyKi2hjyIAwcdI4HRgtvTpIUxBRhB+RFtUh8XtDOxUfctVCnhVi+QQj49i91OyvzkJl6cg==",
"dependencies": {
"glob-to-regexp": "^0.4.1",
"graceful-fs": "^4.1.2"
},
"engines": {
"node": ">=10.13.0"
}
},
"node_modules/which": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/which/-/which-2.0.2.tgz",
+1 -1
View File
@@ -21,7 +21,7 @@
"framer-motion": "^10.16.16",
"fuse.js": "^7.0.0",
"lunr": "^2.3.9",
"next": "14.0.4",
"next": "^14.2.3",
"next-themes": "^0.2.1",
"react": "18.2.0",
"react-dom": "18.2.0",
+1 -2
View File
@@ -2,7 +2,7 @@ import Document, { Head, Html, Main, NextScript } from "next/document";
import React from "react";
// This generates the https: and wss: "connect-src" directives based on the above backends list so its a little easier to edit.
const CSP = `default-src 'none'; script-src 'self' 'unsafe-eval'; img-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; connect-src 'self'`;
const CSP = `default-src 'none'; script-src 'self' 'unsafe-eval'; img-src 'self'; style-src 'self' 'unsafe-inline'; font-src 'self'; frame-src https://youtu.be; connect-src 'self'`;
class MyDocument extends Document {
static async getInitialProps(ctx: any) {
@@ -17,7 +17,6 @@ class MyDocument extends Document {
return (
<Html lang="en">
<Head>
<title>Sliver Docs</title>
<meta httpEquiv="Content-Security-Policy" content={CSP} />
</Head>
<body>
@@ -7,10 +7,10 @@
### Videos
- https://youtu.be/3R6WKUgN0K4?t=456
- https://www.youtube.com/watch?v=QO_1UMaiWHk
- https://www.youtube.com/watch?v=izMMmOaLn9g
- https://www.youtube.com/watch?v=qIbrozlf2wM
- https://www.youtube.com/watch?v=CKfjLnEMfvI
- https://youtu.be/watch?v=QO_1UMaiWHk
- https://youtu.be/watch?v=izMMmOaLn9g
- https://youtu.be/watch?v=qIbrozlf2wM
- https://youtu.be/watch?v=CKfjLnEMfvI
### Blogs
@@ -1,30 +1,22 @@
You'll want to compile from a MacOS or Linux machine, compiling from native Windows in theory is possible, but none of the asset scripts are designed to run on Windows, you can cross-compile the Windows server/client binaries from a better operating system like Linux or MacOS. If you only have a Windows machine see "Windows Builds" below (TL;DR use WSL).
### Compile From Source (No Docker)
# Sliver v1.6.x
From scratch without Docker, requirements for compiling:
- Go v1.21 or later, though we recommend Go v1.22. When compiling v1.5.x use Go v1.20.7
- Go v1.21 or later
- `make`, `sed`, `tar`, `curl`, `zip`, `cut` commands; most of these are installed by default but you may need to install `make`, `curl`, and `zip` depending on your distribution. On MacOS you may need to install XCode and accompanying cli tools.
**IMPORTANT:** The Sliver Makefile requires version information from the git repository, so you must `git clone` the repository. Using GitHub's "download zip" feature may omit the `.git` directory and result in broken builds.
```asciinema
{"src": "/asciinema/compile-from-source.cast", "cols": "132"}
```
#### Compiling
First git clone the repository:
### Compiling
```
$ git clone https://github.com/BishopFox/sliver.git
$ cd sliver
```
The `master` branch will contain the latest Sliver features, however only release version of Sliver are recommended for production use. It is strongly recommended to checkout the latest tagged release branch when compiling from source unless you're a developer:
```
$ git checkout tags/v1.5.39
```
Sliver embeds its own copy of the Go compiler and a few internal tools, the first time your run `make` a bash script will run to download these assets to your local system. This means the first build will take much longer than subsequent builds, especially if your internet connection is slow.
**IMPORTANT:** The Sliver Makefile requires version information from the git repository, so you must `git clone` the repository. Using GitHub's "download zip" feature may omit the `.git` directory and result in broken builds.
By default `make` will build whatever platform you're currently running on:
@@ -34,42 +26,33 @@ $ make
This will create `sliver-server` and `sliver-client` binaries.
#### Cross-compile to Specific Platforms
You can also specify a target platform for the `make` file, though you may need cross-compilers (see below):
```
$ make macos
$ make macos-arm64
$ make linux
$ make linux-arm64
$ make windows
```
### Docker Build
The Docker builds are mostly designed for running unit tests, but can be useful if you want a "just do everything" build, you just need to have Docker installed on the machine. First `git clone` the Sliver repo, then run the `build.py` script (the script isn't required but has a few short cuts see `./build.py --help`). Alternatively, execute the following command:
There are a few Docker targets available depending on your needs:
- `test` - Runs the unit tests
- `production` - Builds the production Docker image, including optional dependencies like Metasploit
- `production-slim` - Builds the production Docker image, but without Metasploit and other optional dependencies
From the project root directory run:
```
docker build -t sliver .
docker build --target production -t sliver .
```
#### Compiling Sliver on Kali Linux
```asciinema
{"src": "/asciinema/sliver-docker-production.cast", "cols": "132"}
```
The Docker build includes mingw and Metasploit, so it can take a while to build from scratch but Docker should cache the layers effectively. Sliver will also run it's unit tests as part of the build, and that takes a few minutes too.
### Windows Builds
**NOTE:** Starting with v1.1.0 in order to cross-compile Windows builds you'll need `mingw` on your system:
- Kali/Ubuntu/Debian `sudo apt install mingw-w64`
- MacOS `brew install mingw-w64`
If all you have is a Windows machine, the easiest way to build Sliver is using [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) and following the Linux/cross-compile instructions above. To cross-compile a native Windows binary use `make windows` and copy it to your Windows file system (i.e. `/mnt/c/Users/foo/Desktop`) and run it using a terminal that supports ANSI sequences such as the [Windows Terminal](https://github.com/microsoft/terminal).
## Developers
# Developers
If you want to modify any of the `.proto` files you'll need to setup a few additional tools to regenerate the `.pb.go` files.
- Protoc v3.19.4 or later
- Protoc libprotoc v26.1 or later
- Protoc-gen-go v1.27.1
- Protoc-gen-go-grpc v1.2.0
@@ -95,3 +78,66 @@ Ensure that these are both on your `$PATH` after running the commands, if not yo
```
$ make pb
```
# Sliver v1.5.x
- When compiling v1.5.x use Go v1.20.7 you can check your version using `go version`. You can use the [Go Version Manager](https://github.com/moovweb/gvm) to install and manage multiple versions of Go.
- `make`, `sed`, `tar`, `curl`, `zip`, `cut` commands; most of these are installed by default but you may need to install `make`, `curl`, and `zip` depending on your distribution. On MacOS you may need to install XCode and accompanying cli tools.
### Compiling
First git clone the repository:
```
$ git clone https://github.com/BishopFox/sliver.git
$ cd sliver
```
The `master` branch will contain the latest Sliver features, however only release version of Sliver are recommended for production use. It is strongly recommended to checkout the latest tagged release branch when compiling from source unless you're a developer:
```
$ git checkout tags/v1.5.42
```
Sliver embeds its own copy of the Go compiler and a few internal tools, the first time your run `make` a bash script will run to download these assets to your local system. This means the first build will take much longer than subsequent builds, especially if your internet connection is slow.
By default `make` will build whatever platform you're currently running on:
```
$ make
```
**IMPORTANT:** The Sliver Makefile requires version information from the git repository, so you must `git clone` the repository. Using GitHub's "download zip" feature may omit the `.git` directory and result in broken builds.
This will create `sliver-server` and `sliver-client` binaries.
### Cross-compile to Specific Platforms
You can also specify a target platform for the `make` file, though you may need cross-compilers (see below):
```
$ make macos
$ make macos-arm64
$ make linux
$ make linux-arm64
$ make windows
```
### Docker Build
The Docker builds are mostly designed for running unit tests, but can be useful if you want a "just do everything" build:
```
docker build -t sliver .
```
The Docker build includes mingw and Metasploit, so it can take a while to build from scratch but Docker should cache the layers effectively. Sliver will also run it's unit tests as part of the build, and that takes a few minutes too.
### Windows Builds
**NOTE:** In versions of Sliver prior to 1.6.0 you will need mingw to cross-compile Windows builds:
- Kali/Ubuntu/Debian `sudo apt install mingw-w64`
- MacOS `brew install mingw-w64`
If all you have is a Windows machine, the easiest way to build Sliver is using [WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10) and following the Linux/cross-compile instructions above. To cross-compile a native Windows binary use `make windows` and copy it to your Windows file system (i.e. `/mnt/c/Users/foo/Desktop`) and run it using a terminal that supports ANSI sequences such as the [Windows Terminal](https://github.com/microsoft/terminal).
@@ -10,7 +10,7 @@ For a Linux server, you can also use the one liner installation `curl https://sl
{"src": "/asciinema/install-1.cast", "cols": "132"}
```
If you install Sliver via the one liner, you can check that the server service is running using `systemctl status sliver`. Note that the Sliver service is not configured to start automatically on boot by default (i.e., if you reboot the server you'll need to start the service again using `systemctrl start sliver`):
If you install Sliver via the one liner, you can check that the server service is running using `systemctl status sliver`. Note that the Sliver service is not configured to start automatically on boot by default (i.e., if you reboot the server you'll need to start the service again using `systemctl start sliver`):
```asciinema
{"src": "/asciinema/service-status-1.cast", "cols": "132", "rows": "14", "idleTimeLimit": 8}
+27 -14
View File
@@ -18,9 +18,8 @@ export default function Home() {
}
return (
<div className="grid grid-cols-12 mt-2">
<div className="col-span-1"></div>
<div className="col-span-5 mr-2">
<div className="grid grid-cols-12 mt-2 gap-2 ml-12 mr-12">
<div className="col-span-6">
<AsciinemaPlayer
src="/asciinema/intro.cast"
rows="18"
@@ -31,7 +30,7 @@ export default function Home() {
loop={true}
/>
</div>
<div className="col-span-5 ml-2">
<div className="col-span-6 ml-2">
<Card>
<CardHeader>
<div className="flex items-center">
@@ -64,21 +63,17 @@ export default function Home() {
</CardBody>
</Card>
</div>
<div className="col-span-1"></div>
<div className="col-span-1"></div>
<div className="col-span-10 mt-8">
<div className="col-span-12 mt-8">
<Divider />
</div>
<div className="col-span-1"></div>
<div className="col-span-1"></div>
<div className="col-span-10 mt-8">
<div className="grid grid-cols-8">
<div className="col-span-2">
<div className="col-span-12 mt-8">
<div className="grid grid-cols-9 gap-2">
<div className="col-span-3">
<TutorialCard
name="Getting Started"
description="A quick start guide to get you up and running with Sliver."
description="A quick start guide to get you up and running"
asciiCast="/asciinema/install-1.cast"
cols="133"
rows="32"
@@ -93,9 +88,27 @@ export default function Home() {
}}
/>
</div>
<div className="col-span-3">
<TutorialCard
name="Compile From Source"
description="How to compile Sliver from source"
asciiCast="/asciinema/compile-from-source.cast"
cols="133"
rows="32"
idleTimeLimit={1}
showButton={true}
buttonText="Read Docs"
onPress={() => {
router.push({
pathname: "/docs",
query: { name: "Compile from Source" },
});
}}
/>
</div>
</div>
</div>
<div className="col-span-1"></div>
<div className="col-span-12 mb-8"></div>
</div>
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+2 -2
View File
@@ -22,8 +22,8 @@ elif command -v yum &> /dev/null; then # Redhat-based OS (Fedora, CentOS, RHEL)
INSTALLER=(yum -y)
elif command -v pacman &>/dev/null; then # Arch-based (Manjaro, Garuda, Blackarch)
echo "Installing dependencies using pacman..."
pacman -S mingw-w64-gcc mingw-w64-binutils mingw-w64-headers
INSTALLER=(pacman -S)
pacman --noconfirm -S mingw-w64-gcc mingw-w64-binutils mingw-w64-headers
INSTALLER=(pacman --noconfirm -S)
else
echo "Unsupported OS, exiting"
exit
+22 -22
View File
@@ -15,17 +15,17 @@ require (
github.com/Binject/universal v0.0.0-20220519011857-bea739e758c0
github.com/Ne0nd0g/go-clr v1.0.3
github.com/alecthomas/chroma v0.10.0
github.com/cheggaaa/pb/v3 v3.1.2
github.com/cheggaaa/pb/v3 v3.1.5
github.com/chromedp/cdproto v0.0.0-20240426225625-909263490071
github.com/chromedp/chromedp v0.9.5
github.com/fatih/color v1.16.0
github.com/glebarez/sqlite v1.10.0
github.com/fatih/color v1.17.0
github.com/glebarez/sqlite v1.11.0
github.com/gofrs/uuid v4.4.0+incompatible
github.com/golang/protobuf v1.5.4
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1
github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
github.com/jedib0t/go-pretty/v6 v6.5.8
github.com/jedib0t/go-pretty/v6 v6.5.9
github.com/kbinani/screenshot v0.0.0-20191211154542-3a185f1ce18f
github.com/klauspost/compress v1.17.8
github.com/lesnuages/go-winio v0.4.19
@@ -34,37 +34,37 @@ require (
github.com/miekg/dns v1.1.59
github.com/moloch--/asciicast v0.1.0
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e
github.com/ncruces/go-sqlite3 v0.7.2
github.com/ncruces/go-sqlite3 v0.15.0
github.com/reeflective/console v0.1.15
github.com/reeflective/readline v1.0.14
github.com/rsteube/carapace v0.46.3-0.20231214181515-27e49f3c3b69
github.com/sirupsen/logrus v1.9.3
github.com/spf13/cobra v1.8.0
github.com/spf13/pflag v1.0.5
github.com/stretchr/testify v1.8.4
github.com/tetratelabs/wazero v1.7.1
github.com/stretchr/testify v1.9.0
github.com/tetratelabs/wazero v1.7.2
github.com/things-go/go-socks5 v0.0.5
github.com/ulikunitz/xz v0.5.12
github.com/xlab/treeprint v1.2.0
github.com/yiya1989/sshkrb5 v0.0.1
golang.org/x/crypto v0.22.0
golang.org/x/crypto v0.23.0
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f
golang.org/x/net v0.24.0
golang.org/x/sys v0.19.0
golang.org/x/term v0.19.0
golang.org/x/text v0.14.0
golang.org/x/net v0.25.0
golang.org/x/sys v0.20.0
golang.org/x/term v0.20.0
golang.org/x/text v0.15.0
golang.zx2c4.com/wireguard v0.0.0-20231211153847-12269c276173
golang.zx2c4.com/wireguard/wgctrl v0.0.0-20230429144221-925a1e7659e6
google.golang.org/grpc v1.63.2
google.golang.org/protobuf v1.34.0
google.golang.org/grpc v1.64.0
google.golang.org/protobuf v1.34.1
gopkg.in/AlecAivazis/survey.v1 v1.8.8
gorm.io/driver/mysql v1.5.6
gorm.io/driver/postgres v1.5.7
gorm.io/driver/sqlite v1.5.5
gorm.io/gorm v1.25.10
gvisor.dev/gvisor v0.0.0-20240306221502-ee1e1f6070e3
modernc.org/sqlite v1.29.8
tailscale.com v1.64.2
modernc.org/sqlite v1.29.10
tailscale.com v1.66.3
)
require (
@@ -95,8 +95,7 @@ require (
github.com/aws/smithy-go v1.19.0 // indirect
github.com/bits-and-blooms/bitset v1.13.0 // indirect
github.com/chromedp/sysutil v1.0.0 // indirect
github.com/coreos/go-iptables v0.7.0 // indirect
github.com/coreos/go-systemd/v22 v22.5.0 // indirect
github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dblohm7/wingoes v0.0.0-20240119213807-a09d6be7affa // indirect
github.com/demisto/goxforce v0.0.0-20160322194047-db8357535b1d // indirect
@@ -107,6 +106,7 @@ require (
github.com/gaissmai/bart v0.4.1 // indirect
github.com/gen2brain/shm v0.0.0-20200228170931-49f9650110c5 // indirect
github.com/glebarez/go-sqlite v1.21.2 // indirect
github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 // indirect
github.com/go-ole/go-ole v1.3.0 // indirect
github.com/go-sql-driver/mysql v1.7.0 // indirect
github.com/gobwas/httphead v0.1.0 // indirect
@@ -116,7 +116,7 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/google/btree v1.1.2 // indirect
github.com/google/go-cmp v0.6.0 // indirect
github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c // indirect
github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 // indirect
github.com/gorilla/csrf v1.7.2 // indirect
github.com/gorilla/securecookie v1.1.2 // indirect
github.com/hashicorp/go-uuid v1.0.2 // indirect
@@ -150,7 +150,7 @@ require (
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/mitchellh/go-ps v1.0.0 // indirect
github.com/ncruces/go-strftime v0.1.9 // indirect
github.com/ncruces/julianday v0.1.5 // indirect
github.com/ncruces/julianday v1.0.0 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
@@ -165,7 +165,7 @@ require (
github.com/tailscale/netlink v1.1.1-0.20211101221916-cabfb018fe85 // indirect
github.com/tailscale/peercred v0.0.0-20240214030740-b535050b2aa4 // indirect
github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 // indirect
github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 // indirect
github.com/tailscale/wireguard-go v0.0.0-20240429185444-03c5a0ccf754 // indirect
github.com/tcnksm/go-httpstat v0.2.0 // indirect
github.com/thedevsaddam/gojsonq/v2 v2.5.2 // indirect
github.com/u-root/uio v0.0.0-20240118234441-a3c409a6018e // indirect
@@ -180,7 +180,7 @@ require (
golang.org/x/tools v0.20.0 // indirect
golang.zx2c4.com/wintun v0.0.0-20230126152724-0fa3db229ce2 // indirect
golang.zx2c4.com/wireguard/windows v0.5.3 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect
gopkg.in/jcmturner/aescts.v1 v1.0.1 // indirect
gopkg.in/jcmturner/dnsutils.v1 v1.0.1 // indirect
gopkg.in/jcmturner/goidentity.v3 v3.0.0 // indirect
+48 -49
View File
@@ -1,7 +1,7 @@
cloud.google.com/go v0.26.0 h1:e0WKqKTd5BnrG8aKH3J3h+QvEIQtSUcf2n5UZ5ZgLtQ=
cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
cloud.google.com/go/compute v1.24.0 h1:phWcR2eWzRJaL/kOiJwfFsPs4BaKq1j6vnpZrc1YlVg=
cloud.google.com/go/compute v1.24.0/go.mod h1:kw1/T+h/+tK2LJK0wiPPx1intgdAM3j/g3hFDlscY40=
cloud.google.com/go/compute v1.25.1 h1:ZRpHJedLtTpKgr3RV1Fx23NuaAEN1Zfx9hw1u4aJdjU=
cloud.google.com/go/compute v1.25.1/go.mod h1:oopOIR53ly6viBYxaDhBfJwzUAxf1zE//uf3IB011ls=
cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY=
cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA=
filippo.io/age v1.1.1 h1:pIpO7l151hCnQ4BdyBujnGP2YlUo0uj6sAVNHGBvXHg=
@@ -86,8 +86,8 @@ github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZx
github.com/bits-and-blooms/bitset v1.13.0 h1:bAQ9OPNFYbGHV6Nez0tmNI0RiEu7/hxlYJRUA0wFAVE=
github.com/bits-and-blooms/bitset v1.13.0/go.mod h1:7hO7Gc7Pp1vODcmWvKMRA9BNmbv6a/7QIWpPxHddWR8=
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
github.com/cheggaaa/pb/v3 v3.1.2 h1:FIxT3ZjOj9XJl0U4o2XbEhjFfZl7jCVCDOGq1ZAB7wQ=
github.com/cheggaaa/pb/v3 v3.1.2/go.mod h1:SNjnd0yKcW+kw0brSusraeDd5Bf1zBfxAzTL2ss3yQ4=
github.com/cheggaaa/pb/v3 v3.1.5 h1:QuuUzeM2WsAqG2gMqtzaWithDJv0i+i6UlnwSCI4QLk=
github.com/cheggaaa/pb/v3 v3.1.5/go.mod h1:CrxkeghYTXi1lQBEI7jSn+3svI3cuc19haAj6jM60XI=
github.com/chromedp/cdproto v0.0.0-20240202021202-6d0b6a386732/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
github.com/chromedp/cdproto v0.0.0-20240426225625-909263490071 h1:RdCf9hH3xq5vJifrjGB7zQlFkdRB3pAppcX2helDq2U=
github.com/chromedp/cdproto v0.0.0-20240426225625-909263490071/go.mod h1:GKljq0VrfU4D5yc+2qA6OVr8pmO/MBbPEWqWQ/oqGEs=
@@ -99,10 +99,8 @@ github.com/cilium/ebpf v0.12.3 h1:8ht6F9MquybnY97at+VDZb3eQQr8ev79RueWeVaEcG4=
github.com/cilium/ebpf v0.12.3/go.mod h1:TctK1ivibvI3znr66ljgi4hqOT8EYQjz1KWBfb1UVgM=
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
github.com/coreos/go-iptables v0.7.0 h1:XWM3V+MPRr5/q51NuWSgU0fqMad64Zyxs8ZUoMsamr8=
github.com/coreos/go-iptables v0.7.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs=
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6 h1:8h5+bWd7R6AYUslN6c6iuZWTKsKxUFDlpnmilO6R2n0=
github.com/coreos/go-iptables v0.7.1-0.20240112124308-65c67c9f46e6/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFETJConOQ//Q=
github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o=
github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4=
github.com/creack/pty v1.1.21 h1:1/QdRyBaHHJP61QkWMXlOIBfsgdDeeKfK8SYVUWJKf0=
@@ -129,8 +127,8 @@ github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.m
github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
github.com/fatih/color v1.12.0/go.mod h1:ELkj/draVOlAH/xkhN6mQ50Qd0MPOk5AAr3maGEBuJM=
github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM=
github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE=
github.com/fatih/color v1.17.0 h1:GlRw1BRJxkpqUCBKzKOw098ed57fEsKeNjpTe3cSjK4=
github.com/fatih/color v1.17.0/go.mod h1:YZ7TlrGPkiz6ku9fK3TLD/pl3CpsiFyu8N92HLgmosI=
github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8=
github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0=
github.com/fxamacker/cbor/v2 v2.5.0 h1:oHsG0V/Q6E/wqTS2O1Cozzsy69nqCiguo5Q1a1ADivE=
@@ -143,8 +141,10 @@ github.com/github/fakeca v0.1.0 h1:Km/MVOFvclqxPM9dZBC4+QE564nU4gz4iZ0D9pMw28I=
github.com/github/fakeca v0.1.0/go.mod h1:+bormgoGMMuamOscx7N91aOuUST7wdaJ2rNjeohylyo=
github.com/glebarez/go-sqlite v1.21.2 h1:3a6LFC4sKahUunAmynQKLZceZCOzUthkRkEAl9gAXWo=
github.com/glebarez/go-sqlite v1.21.2/go.mod h1:sfxdZyhQjTM2Wry3gVYWaW072Ri1WMdWJi0k6+3382k=
github.com/glebarez/sqlite v1.10.0 h1:u4gt8y7OND/cCei/NMHmfbLxF6xP2wgKcT/BJf2pYkc=
github.com/glebarez/sqlite v1.10.0/go.mod h1:IJ+lfSOmiekhQsFTJRx/lHtGYmCdtAiTaf5wI9u5uHA=
github.com/glebarez/sqlite v1.11.0 h1:wSG0irqzP6VurnMEpFGer5Li19RpIRi2qvQz++w0GMw=
github.com/glebarez/sqlite v1.11.0/go.mod h1:h8/o8j5wiAsqSPoWELDUdJXhjAhsVliSn7bWZjOhrgQ=
github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0 h1:ymLjT4f35nQbASLnvxEde4XOBL+Sn7rFuV+FOJqkljg=
github.com/go-json-experiment/json v0.0.0-20231102232822-2e55bd4e08b0/go.mod h1:6daplAwHHGbUGib4990V3Il26O0OC4aRyvewaaAihaA=
github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY=
github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A=
github.com/go-ole/go-ole v1.3.0 h1:Dt6ye7+vXGIKZ7Xtk4s6/xVdGDQynvom7xCFEdWr6uE=
@@ -158,7 +158,6 @@ github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og=
github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw=
github.com/gobwas/ws v1.3.2 h1:zlnbNHxumkRvfPWgfXu8RBwyNR1x8wh9cf5PTOCqs9Q=
github.com/gobwas/ws v1.3.2/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466 h1:sQspH8M4niEijh3PFscJRLDnkL547IeP7kpPe3uUhEg=
github.com/godbus/dbus/v5 v5.1.1-0.20230522191255-76236955d466/go.mod h1:ZiQxhyQ+bbbfxUKVvjfO498oPYvtYhZzycal3G/NHmU=
github.com/gofrs/uuid v4.4.0+incompatible h1:3qXRTX8/NbyulANqlc0lchS1gqAVxRgsuW1YrTJupqA=
@@ -181,8 +180,8 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0=
github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c h1:06RMfw+TMMHtRuUOroMeatRCCgSMWXCJQeABvHU69YQ=
github.com/google/nftables v0.1.1-0.20230115205135-9aa6fdf5a28c/go.mod h1:BVIYo3cdnT4qSylnYqcd5YtmXhr51cJPGtnLBe/uLBU=
github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806 h1:wG8RYIyctLhdFk6Vl1yPGtSRtwGpVkWyZww1OCil2MI=
github.com/google/nftables v0.2.1-0.20240414091927-5e242ec57806/go.mod h1:Beg6V6zZ3oEn0JuiUQ4wqwuyqqzasOltcoXPtgLbFp4=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd h1:gbpYu9NMq8jhDVbvlGkMFWCjLFlqqEZjEmObmhUy6Vo=
github.com/google/pprof v0.0.0-20240409012703-83162a5b38cd/go.mod h1:kf6iHlnVGwgKolg33glAes7Yg/8iWP8ukqeldJSO7jw=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
@@ -221,8 +220,8 @@ github.com/jackc/puddle/v2 v2.2.1 h1:RhxXJtFG022u4ibrCSMSiu5aOq1i77R3OHKNJj77OAk
github.com/jackc/puddle/v2 v2.2.1/go.mod h1:vriiEXHvEE654aYKXXjOvZM39qJ0q+azkZFrfEOc3H4=
github.com/jcmturner/gofork v1.0.0 h1:J7uCkflzTEhUZ64xqKnkDxq3kzc96ajM1Gli5ktUem8=
github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
github.com/jedib0t/go-pretty/v6 v6.5.8 h1:8BCzJdSvUbaDuRba4YVh+SKMGcAAKdkcF3SVFbrHAtQ=
github.com/jedib0t/go-pretty/v6 v6.5.8/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/jedib0t/go-pretty/v6 v6.5.9 h1:ACteMBRrrmm1gMsXe9PSTOClQ63IXDUt03H5U+UV8OU=
github.com/jedib0t/go-pretty/v6 v6.5.9/go.mod h1:zbn98qrYlh95FIhwwsbIip0LYpwSG8SUOScs+v9/t0E=
github.com/jellydator/ttlcache/v3 v3.1.0 h1:0gPFG0IHHP6xyUyXq+JaD8fwkDCqgqwohXNJBcYE71g=
github.com/jellydator/ttlcache/v3 v3.1.0/go.mod h1:hi7MGFdMAwZna5n2tuvh63DvFLzVKySzCVW6+0gA2n4=
github.com/jinzhu/inflection v1.0.0 h1:K317FqzuhWc8YvSVlFMCCUb36O/S9MCKRDI7QkRKD/E=
@@ -303,12 +302,12 @@ github.com/moloch--/asciicast v0.1.0 h1:eBOJwuFKSk447s/kPs9MWsc4kAl5HmuKIDLDYD6/
github.com/moloch--/asciicast v0.1.0/go.mod h1:OckO16UDLgxVLclrCnbocL1ix15Br/8Xv/caBoYq98o=
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e h1:IkFCPlAa0iTiLxck+NqAwBx8JDlnHYm4orOQBbs4BDQ=
github.com/moloch--/memmod v0.0.0-20230225130813-fd77d905589e/go.mod h1:eYeI6cQ5YHhHt9i0BBW0zc1DaQnb4ZMXsSPuEV/V5Og=
github.com/ncruces/go-sqlite3 v0.7.2 h1:K7jU4rnUxFdUsbEL+B0Xc+VexLTEwGSO6Qh91Qh4hYc=
github.com/ncruces/go-sqlite3 v0.7.2/go.mod h1:t3dP4AP9rJddU+ffFv0h6fWyeOCEhjxrYc1nsYG7aQI=
github.com/ncruces/go-sqlite3 v0.15.0 h1:C+SIrcYsAIR5GUYWmCnif6x81n6BS9y75vYcQynuGNU=
github.com/ncruces/go-sqlite3 v0.15.0/go.mod h1:kHHYmFmK4G2VFFoIovEg9BEQ8BP+D81y4ESHXnzJV/w=
github.com/ncruces/go-strftime v0.1.9 h1:bY0MQC28UADQmHmaF5dgpLmImcShSi2kHU9XLdhx/f4=
github.com/ncruces/go-strftime v0.1.9/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/ncruces/julianday v0.1.5 h1:hDJ9ejiMp3DHsoZ5KW4c1lwfMjbARS7u/gbYcd0FBZk=
github.com/ncruces/julianday v0.1.5/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/ncruces/julianday v1.0.0 h1:fH0OKwa7NWvniGQtxdJRxAgkBMolni2BjDHaWTxqt7M=
github.com/ncruces/julianday v1.0.0/go.mod h1:Dusn2KvZrrovOMJuOt0TNXL6tB7U2E8kvza5fFc9G7g=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646 h1:zYyBkD/k9seD2A7fsi6Oo2LfFZAehjjQMERAvZLEDnQ=
github.com/nfnt/resize v0.0.0-20180221191011-83c6a9932646/go.mod h1:jpp1/29i3P1S/RLdc7JQKbRpFeM1dOBd8T9ki5s+AY8=
github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o=
@@ -358,8 +357,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.9.0 h1:HtqpIVDClZ4nwg75+f6Lvsy/wHu+3BoSGCbBAcpTsTg=
github.com/stretchr/testify v1.9.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e h1:PtWT87weP5LWHEY//SWsYkSO3RWRZo4OSWagh3YD2vQ=
github.com/tailscale/certstore v0.1.1-0.20231202035212-d3fa0460f47e/go.mod h1:XrBNfAFN+pwoWuksbFS9Ccxnopa15zJGgXRFN90l3K4=
github.com/tailscale/go-winio v0.0.0-20231025203758-c4f33415bf55 h1:Gzfnfk2TWrk8Jj4P4c1a3CtQyMaTVCznlkLZI++hok4=
@@ -378,16 +377,16 @@ github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1 h1:t
github.com/tailscale/web-client-prebuilt v0.0.0-20240226180453-5db17b287bf1/go.mod h1:agQPE6y6ldqCOui2gkIh7ZMztTkIQKH049tv8siLuNQ=
github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6 h1:l10Gi6w9jxvinoiq15g8OToDdASBni4CyJOdHY1Hr8M=
github.com/tailscale/wf v0.0.0-20240214030419-6fbb0a674ee6/go.mod h1:ZXRML051h7o4OcI0d3AaILDIad/Xw0IkXaHM17dic1Y=
github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272 h1:zwsem4CaamMdC3tFoTpzrsUSMDPV0K6rhnQdF7kXekQ=
github.com/tailscale/wireguard-go v0.0.0-20231121184858-cc193a0b3272/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
github.com/tailscale/wireguard-go v0.0.0-20240429185444-03c5a0ccf754 h1:iazWjqVHE6CbNam7WXRhi33Qad5o7a8LVYgVoILpZdI=
github.com/tailscale/wireguard-go v0.0.0-20240429185444-03c5a0ccf754/go.mod h1:BOm5fXUBFM+m9woLNBoxI9TaBXXhGNP50LX/TGIvGb4=
github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9 h1:81P7rjnikHKTJ75EkjppvbwUfKHDHYk6LJpO5PZy8pA=
github.com/tailscale/xnet v0.0.0-20240117122442-62b9a7c569f9/go.mod h1:orPd6JZXXRyuDusYilywte7k094d7dycXXU5YnWsrwg=
github.com/tc-hib/winres v0.2.1 h1:YDE0FiP0VmtRaDn7+aaChp1KiF4owBiJa5l964l5ujA=
github.com/tc-hib/winres v0.2.1/go.mod h1:C/JaNhH3KBvhNKVbvdlDWkbMDO9H4fKKDaN7/07SSuk=
github.com/tcnksm/go-httpstat v0.2.0 h1:rP7T5e5U2HfmOBmZzGgGZjBQ5/GluWUylujl0tJ04I0=
github.com/tcnksm/go-httpstat v0.2.0/go.mod h1:s3JVJFtQxtBEBC9dwcdTTXS9xFnM3SXAZwPG41aurT8=
github.com/tetratelabs/wazero v1.7.1 h1:QtSfd6KLc41DIMpDYlJdoMc6k7QTN246DM2+n2Y/Dx8=
github.com/tetratelabs/wazero v1.7.1/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/tetratelabs/wazero v1.7.2 h1:1+z5nXJNwMLPAWaTePFi49SSTL0IMx/i3Fg8Yc25GDc=
github.com/tetratelabs/wazero v1.7.2/go.mod h1:ytl6Zuh20R/eROuyDaGPkp82O9C/DJfXAwJfQ3X6/7Y=
github.com/thedevsaddam/gojsonq/v2 v2.5.2 h1:CoMVaYyKFsVj6TjU6APqAhAvC07hTI6IQen8PHzHYY0=
github.com/thedevsaddam/gojsonq/v2 v2.5.2/go.mod h1:bv6Xa7kWy82uT0LnXPE2SzGqTj33TAEeR560MdJkiXs=
github.com/things-go/go-socks5 v0.0.5 h1:qvKaGcBkfDrUL33SchHN93srAmYGzb4CxSM2DPYufe8=
@@ -426,8 +425,8 @@ golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8U
golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20201016220609-9e8e0b390897/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30=
golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M=
golang.org/x/crypto v0.23.0 h1:dIJU/v2J8Mdglj/8rJ6UUOM3Zc9zLZxVZwwxMooUSAI=
golang.org/x/crypto v0.23.0/go.mod h1:CKFgDieR+mRhux2Lsu27y0fO304Db0wZe70UKqHu0v8=
golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f h1:99ci1mjWVBWwJiEKYY6jWa4d2nTQVIEhZIptnrVb1XY=
golang.org/x/exp v0.0.0-20240416160154-fe59bbe5cc7f/go.mod h1:/lliqkxwWAhPjf5oSOIJup2XcqJaw8RGS6k3TGEc7GI=
@@ -454,11 +453,11 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w=
golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8=
golang.org/x/net v0.25.0 h1:d/OCCoBEUq33pjydKrGQhw7IlUPI2Oylr+8qLx49kac=
golang.org/x/net v0.25.0/go.mod h1:JkAGAh7GEvH74S6FOH42FLoXpXbE/aqXSrIQjXgsiwM=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.17.0 h1:6m3ZPmLEFdVxKKWnKq4VqZ60gutO35zm+zrAHVmHyDQ=
golang.org/x/oauth2 v0.17.0/go.mod h1:OzPDGQiuQMguemayvdylqddI7qcD9lnSDb+1FiwQ5HA=
golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI=
golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -497,18 +496,18 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.4.1-0.20230131160137-e7d7f63158de/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o=
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/sys v0.20.0 h1:Od9JTbYCk261bKm4M/mw7AklTlFYIa0bIp9BgSm1S8Y=
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q=
golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk=
golang.org/x/term v0.20.0 h1:VnkxpohqXaOBYJtBmEppKUG6mXpi+4O6purfc2+sMhw=
golang.org/x/term v0.20.0/go.mod h1:8UkIAJTvZgivsXaD6/pH6U9ecQzZ45awqEOzuCvwpFY=
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ=
golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/text v0.15.0 h1:h1V/4gjBv8v9cjcR6+AR5+/cIYK5N/WAgiv4xlsEtAk=
golang.org/x/text v0.15.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU=
golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
@@ -542,17 +541,17 @@ google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJ
google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de h1:cZGRis4/ot9uVm639a+rHCUaG0JJHEsdyzSQTMX+suY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY=
google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk=
google.golang.org/grpc v1.63.2 h1:MUeiw1B2maTVZthpU5xvASfTh3LDbxHd6IJ6QQVU+xM=
google.golang.org/grpc v1.63.2/go.mod h1:WAX/8DgncnokcFUldAxq7GeB5DXHDbMF+lLvDomNkRA=
google.golang.org/protobuf v1.34.0 h1:Qo/qEd2RZPCf2nKuorzksSknv0d3ERwp1vFG38gSmH4=
google.golang.org/protobuf v1.34.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY=
google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg=
google.golang.org/protobuf v1.34.1 h1:9ddQBjfCyZPOHPUiPxpYESBLc+T8P3E+Vo4IbKZgFWg=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
gopkg.in/AlecAivazis/survey.v1 v1.8.8 h1:5UtTowJZTz1j7NxVzDGKTz6Lm9IWm8DDF6b7a2wq9VY=
gopkg.in/AlecAivazis/survey.v1 v1.8.8/go.mod h1:CaHjv79TCgAvXMSFJSVgonHXYWxnhzI3eoHtnX5UgUo=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
@@ -614,8 +613,8 @@ modernc.org/opt v0.1.3 h1:3XOZf2yznlhC+ibLltsDGzABUGVx8J6pnFMS3E4dcq4=
modernc.org/opt v0.1.3/go.mod h1:WdSiB5evDcignE70guQKxYUl14mgWtbClRi5wmkkTX0=
modernc.org/sortutil v1.2.0 h1:jQiD3PfS2REGJNzNCMMaLSp/wdMNieTbKX920Cqdgqc=
modernc.org/sortutil v1.2.0/go.mod h1:TKU2s7kJMf1AE84OoiGppNHJwvB753OYfNl2WRb++Ss=
modernc.org/sqlite v1.29.8 h1:nGKglNx9K5v0As+zF0/Gcl1kMkmaU1XynYyq92PbsC8=
modernc.org/sqlite v1.29.8/go.mod h1:lQPm27iqa4UNZpmr4Aor0MH0HkCLbt1huYDfWylLZFk=
modernc.org/sqlite v1.29.10 h1:3u93dz83myFnMilBGCOLbr+HjklS6+5rJLx4q86RDAg=
modernc.org/sqlite v1.29.10/go.mod h1:ItX2a1OVGgNsFh6Dv60JQvGfJfTPHPVpV6DF59akYOA=
modernc.org/strutil v1.2.0 h1:agBi9dp1I+eOnxXeiZawM8F4LawKv4NzGWSaLfyeNZA=
modernc.org/strutil v1.2.0/go.mod h1:/mdcBmfOibveCTBxUl5B5l6W+TTH1FXPLHZE6bTosX0=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
@@ -626,5 +625,5 @@ nhooyr.io/websocket v1.8.10 h1:mv4p+MnGrLDcPlBoWsvPP7XCzTYMXP9F9eIGoKbgx7Q=
nhooyr.io/websocket v1.8.10/go.mod h1:rN9OFWIUwuxg4fR5tELlYC04bXYowCP9GX47ivo2l+c=
software.sslmate.com/src/go-pkcs12 v0.4.0 h1:H2g08FrTvSFKUj+D309j1DPfk5APnIdAQAB8aEykJ5k=
software.sslmate.com/src/go-pkcs12 v0.4.0/go.mod h1:Qiz0EyvDRJjjxGyUQa2cCNZn/wMyzrRJ/qcDXOQazLI=
tailscale.com v1.64.2 h1:0VNwUsjK6CwgkqyaOANndBER2SMYl8JZ5uNRTvIqCnY=
tailscale.com v1.64.2/go.mod h1:6kGByHNxnFfK1i4gVpdtvpdS1HicHohWXnsfwmXy64I=
tailscale.com v1.66.3 h1:jpWat+hiobTtCosSV/c8D6S/ubgROf/S59MaIBdM9pY=
tailscale.com v1.66.3/go.mod h1:99BIV4U3UPw36Sva04xK2ZsEpVRUkY9jCdEDSAhaNGM=
+30 -6
View File
@@ -155,7 +155,7 @@ func dirListHandler(data []byte, resp RPCResponse) {
path, filter := determineDirPathFilter(targetPath)
dir, files, err := getDirList(path)
dir, rootDirEntry, files, err := getDirList(path)
// Convert directory listing to protobuf
timezone, offset := time.Now().Zone()
@@ -166,6 +166,19 @@ func dirListHandler(data []byte, resp RPCResponse) {
dirList.Exists = false
}
dirList.Files = []*sliverpb.FileInfo{}
rootDirInfo, err := rootDirEntry.Info()
if err == nil && filter == "" {
// We should not get an error because we created the DirEntry object from the FileInfo object
dirList.Files = append(dirList.Files, &sliverpb.FileInfo{
Name: ".", // Cannot use the name from the FileInfo / DirEntry because that is the name of the directory
Size: rootDirInfo.Size(),
ModTime: rootDirInfo.ModTime().Unix(),
Mode: rootDirInfo.Mode().String(),
Uid: getUid(rootDirInfo),
Gid: getGid(rootDirInfo),
IsDir: rootDirInfo.IsDir(),
})
}
var match bool = false
var linkPath string = ""
@@ -226,19 +239,30 @@ func dirListHandler(data []byte, resp RPCResponse) {
resp(data, err)
}
func getDirList(target string) (string, []fs.DirEntry, error) {
func getDirList(target string) (string, fs.DirEntry, []fs.DirEntry, error) {
dir, err := filepath.Abs(target)
if err != nil {
// {{if .Config.Debug}}
log.Printf("dir list failed to construct path %s", err)
// {{end}}
return "", nil, err
return "", nil, nil, err
}
if _, err := os.Stat(dir); !os.IsNotExist(err) {
if rootInfo, err := os.Stat(dir); !os.IsNotExist(err) {
/*
We could place the entry for the directory itself
at the beginning of the returned slice of DirEntry
objects, but then it is not clear if that is the
root directory or a directory / file in the root
directory with the same name as the root directory.
Using WalkDir is not great here because you cannot
tell it to not be recursive, so we will be wasting
cycles telling it to skip directories and files
*/
files, err := os.ReadDir(dir)
return dir, files, err
return dir, fs.FileInfoToDirEntry(rootInfo), files, err
}
return dir, []fs.DirEntry{}, errors.New("directory does not exist")
return dir, nil, []fs.DirEntry{}, errors.New("directory does not exist")
}
func rmHandler(data []byte, resp RPCResponse) {
+15 -2
View File
@@ -31,8 +31,8 @@ import (
"time"
"unsafe"
"github.com/bishopfox/sliver/implant/sliver/procdump"
"github.com/bishopfox/sliver/implant/sliver/mount"
"github.com/bishopfox/sliver/implant/sliver/procdump"
"github.com/bishopfox/sliver/implant/sliver/taskrunner"
"github.com/bishopfox/sliver/protobuf/commonpb"
"github.com/bishopfox/sliver/protobuf/sliverpb"
@@ -190,7 +190,7 @@ func memfilesListHandler(_ []byte, resp RPCResponse) {
pid := os.Getpid()
path := fmt.Sprintf("/proc/%d/fd/", pid)
dir, files, err := getDirList(path)
dir, rootDirEntry, files, err := getDirList(path)
// Convert directory listing to protobuf
timezone, offset := time.Now().Zone()
@@ -201,6 +201,19 @@ func memfilesListHandler(_ []byte, resp RPCResponse) {
dirList.Exists = false
}
dirList.Files = []*sliverpb.FileInfo{}
rootDirInfo, err := rootDirEntry.Info()
if err == nil {
// We should not get an error because we created the DirEntry object from the FileInfo object
dirList.Files = append(dirList.Files, &sliverpb.FileInfo{
Name: ".", // Cannot use the name from the FileInfo / DirEntry because that is the name of the directory
Size: rootDirInfo.Size(),
ModTime: rootDirInfo.ModTime().Unix(),
Mode: rootDirInfo.Mode().String(),
Uid: getUid(rootDirInfo),
Gid: getGid(rootDirInfo),
IsDir: rootDirInfo.IsDir(),
})
}
for _, dirEntry := range files {
//log.Printf("File: %s\n", dirEntry.Name())
File diff suppressed because it is too large. Load diff
+22
View File
@@ -225,6 +225,7 @@ message ExternalImplantConfig {
ImplantConfig Config = 1;
ImplantBuild Build = 2;
HTTPC2Config HTTPC2 = 3;
map<string, uint64> encoders = 4;
}
message ExternalImplantBinary {
@@ -1308,4 +1309,25 @@ message ResourceID {
string Type = 2;
string Name = 3;
uint64 Value = 4;
}
// [ Certificates ] ----------------------------------------
message CertificatesReq {
uint32 CategoryFilters = 1;
string CN = 2;
}
message CertificateData {
string CN = 1;
string CreationTime = 2;
string ValidityStart = 3;
string ValidityExpiry = 4;
string Type = 5;
string KeyAlgorithm = 6;
string ID = 7;
}
message CertificateInfo {
repeated CertificateData info = 1;
}
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc-gen-go v1.34.1
// protoc v5.26.1
// source: commonpb/common.proto
+1 -1
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc-gen-go v1.34.1
// protoc v5.26.1
// source: dnspb/dns.proto
File diff suppressed because it is too large. Load diff
+3
View File
@@ -106,6 +106,9 @@ service SliverRPC {
rpc BuilderTrigger(clientpb.Event) returns (commonpb.Empty);
rpc Builders(commonpb.Empty) returns (clientpb.Builders);
// *** Certificates ***
rpc GetCertificateInfo(clientpb.CertificatesReq) returns (clientpb.CertificateInfo);
// *** Crackstation ***
rpc CrackstationRegister(clientpb.Crackstation)
returns (stream clientpb.Event);
+39
View File
@@ -79,6 +79,7 @@ const (
SliverRPC_BuilderRegister_FullMethodName = "/rpcpb.SliverRPC/BuilderRegister"
SliverRPC_BuilderTrigger_FullMethodName = "/rpcpb.SliverRPC/BuilderTrigger"
SliverRPC_Builders_FullMethodName = "/rpcpb.SliverRPC/Builders"
SliverRPC_GetCertificateInfo_FullMethodName = "/rpcpb.SliverRPC/GetCertificateInfo"
SliverRPC_CrackstationRegister_FullMethodName = "/rpcpb.SliverRPC/CrackstationRegister"
SliverRPC_CrackstationTrigger_FullMethodName = "/rpcpb.SliverRPC/CrackstationTrigger"
SliverRPC_CrackstationBenchmark_FullMethodName = "/rpcpb.SliverRPC/CrackstationBenchmark"
@@ -282,6 +283,8 @@ type SliverRPCClient interface {
BuilderRegister(ctx context.Context, in *clientpb.Builder, opts ...grpc.CallOption) (SliverRPC_BuilderRegisterClient, error)
BuilderTrigger(ctx context.Context, in *clientpb.Event, opts ...grpc.CallOption) (*commonpb.Empty, error)
Builders(ctx context.Context, in *commonpb.Empty, opts ...grpc.CallOption) (*clientpb.Builders, error)
// *** Certificates ***
GetCertificateInfo(ctx context.Context, in *clientpb.CertificatesReq, opts ...grpc.CallOption) (*clientpb.CertificateInfo, error)
// *** Crackstation ***
CrackstationRegister(ctx context.Context, in *clientpb.Crackstation, opts ...grpc.CallOption) (SliverRPC_CrackstationRegisterClient, error)
CrackstationTrigger(ctx context.Context, in *clientpb.Event, opts ...grpc.CallOption) (*commonpb.Empty, error)
@@ -990,6 +993,15 @@ func (c *sliverRPCClient) Builders(ctx context.Context, in *commonpb.Empty, opts
return out, nil
}
func (c *sliverRPCClient) GetCertificateInfo(ctx context.Context, in *clientpb.CertificatesReq, opts ...grpc.CallOption) (*clientpb.CertificateInfo, error) {
out := new(clientpb.CertificateInfo)
err := c.cc.Invoke(ctx, SliverRPC_GetCertificateInfo_FullMethodName, in, out, opts...)
if err != nil {
return nil, err
}
return out, nil
}
func (c *sliverRPCClient) CrackstationRegister(ctx context.Context, in *clientpb.Crackstation, opts ...grpc.CallOption) (SliverRPC_CrackstationRegisterClient, error) {
stream, err := c.cc.NewStream(ctx, &SliverRPC_ServiceDesc.Streams[2], SliverRPC_CrackstationRegister_FullMethodName, opts...)
if err != nil {
@@ -2273,6 +2285,8 @@ type SliverRPCServer interface {
BuilderRegister(*clientpb.Builder, SliverRPC_BuilderRegisterServer) error
BuilderTrigger(context.Context, *clientpb.Event) (*commonpb.Empty, error)
Builders(context.Context, *commonpb.Empty) (*clientpb.Builders, error)
// *** Certificates ***
GetCertificateInfo(context.Context, *clientpb.CertificatesReq) (*clientpb.CertificateInfo, error)
// *** Crackstation ***
CrackstationRegister(*clientpb.Crackstation, SliverRPC_CrackstationRegisterServer) error
CrackstationTrigger(context.Context, *clientpb.Event) (*commonpb.Empty, error)
@@ -2588,6 +2602,9 @@ func (UnimplementedSliverRPCServer) BuilderTrigger(context.Context, *clientpb.Ev
func (UnimplementedSliverRPCServer) Builders(context.Context, *commonpb.Empty) (*clientpb.Builders, error) {
return nil, status.Errorf(codes.Unimplemented, "method Builders not implemented")
}
func (UnimplementedSliverRPCServer) GetCertificateInfo(context.Context, *clientpb.CertificatesReq) (*clientpb.CertificateInfo, error) {
return nil, status.Errorf(codes.Unimplemented, "method GetCertificateInfo not implemented")
}
func (UnimplementedSliverRPCServer) CrackstationRegister(*clientpb.Crackstation, SliverRPC_CrackstationRegisterServer) error {
return status.Errorf(codes.Unimplemented, "method CrackstationRegister not implemented")
}
@@ -4010,6 +4027,24 @@ func _SliverRPC_Builders_Handler(srv interface{}, ctx context.Context, dec func(
return interceptor(ctx, in, info, handler)
}
func _SliverRPC_GetCertificateInfo_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
in := new(clientpb.CertificatesReq)
if err := dec(in); err != nil {
return nil, err
}
if interceptor == nil {
return srv.(SliverRPCServer).GetCertificateInfo(ctx, in)
}
info := &grpc.UnaryServerInfo{
Server: srv,
FullMethod: SliverRPC_GetCertificateInfo_FullMethodName,
}
handler := func(ctx context.Context, req interface{}) (interface{}, error) {
return srv.(SliverRPCServer).GetCertificateInfo(ctx, req.(*clientpb.CertificatesReq))
}
return interceptor(ctx, in, info, handler)
}
func _SliverRPC_CrackstationRegister_Handler(srv interface{}, stream grpc.ServerStream) error {
m := new(clientpb.Crackstation)
if err := stream.RecvMsg(m); err != nil {
@@ -6491,6 +6526,10 @@ var SliverRPC_ServiceDesc = grpc.ServiceDesc{
MethodName: "Builders",
Handler: _SliverRPC_Builders_Handler,
},
{
MethodName: "GetCertificateInfo",
Handler: _SliverRPC_GetCertificateInfo_Handler,
},
{
MethodName: "CrackstationTrigger",
Handler: _SliverRPC_CrackstationTrigger_Handler,
+7 -4
View File
@@ -1,6 +1,6 @@
// Code generated by protoc-gen-go. DO NOT EDIT.
// versions:
// protoc-gen-go v1.27.1
// protoc-gen-go v1.34.1
// protoc v5.26.1
// source: sliverpb/sliver.proto
@@ -173,7 +173,8 @@ func (PeerFailureType) EnumDescriptor() ([]byte, []int) {
}
// Envelope - Used to encode implant<->server messages since we
// cannot use gRPC due to the various transports used.
//
// cannot use gRPC due to the various transports used.
type Envelope struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -746,7 +747,8 @@ func (x *CloseSession) GetRequest() *commonpb.Request {
}
// Ping - Not ICMP, just sends a rount trip message to an implant to
// see if it's still responding.
//
// see if it's still responding.
type Ping struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
@@ -3684,7 +3686,8 @@ func (x *CurrentTokenOwner) GetResponse() *commonpb.Response {
}
// InvokeGetSystemReq - Implant-side version of GetSystemReq, this message
// contains the .Data based on the client's req.Config
//
// contains the .Data based on the client's req.Config
type InvokeGetSystemReq struct {
state protoimpl.MessageState
sizeCache protoimpl.SizeCache
+8
View File
@@ -33,6 +33,7 @@ import (
"github.com/bishopfox/sliver/protobuf/rpcpb"
"github.com/bishopfox/sliver/server/db"
"github.com/bishopfox/sliver/server/db/models"
"github.com/bishopfox/sliver/server/encoders"
"github.com/bishopfox/sliver/server/generate"
"github.com/bishopfox/sliver/server/log"
"google.golang.org/grpc"
@@ -188,6 +189,13 @@ func (b *Builder) handleBuildEvent(event *clientpb.Event) {
})
return
}
encoders.Base32EncoderID = extConfig.Encoders["base32"]
encoders.Base58EncoderID = extConfig.Encoders["base58"]
encoders.Base64EncoderID = extConfig.Encoders["base64"]
encoders.EnglishEncoderID = extConfig.Encoders["english"]
encoders.GzipEncoderID = extConfig.Encoders["gzip"]
encoders.HexEncoderID = extConfig.Encoders["hex"]
encoders.PNGEncoderID = extConfig.Encoders["png"]
var fPath string
switch extConfig.Config.Format {
+10 -1
View File
@@ -810,7 +810,16 @@ func newHTTPSessionID() string {
func getRemoteAddr(req *http.Request) string {
ipAddress := req.Header.Get("X-Real-Ip")
if ipAddress == "" {
ipAddress = req.Header.Get("X-Forwarded-For")
xForwardedFor := req.Header.Get("X-Forwarded-For")
if xForwardedFor != "" {
ips := strings.Split(xForwardedFor, ",")
if len(ips) > 0 {
// Extracts original client ip address
ipAddress = strings.TrimSpace(ips[0])
} else {
ipAddress = xForwardedFor
}
}
}
if ipAddress == "" {
return req.RemoteAddr
+1
View File
@@ -58,6 +58,7 @@ var (
ErrNonUniqueFileExt = errors.New("implant config must specify unique file extensions")
ErrQueryParamNameLen = errors.New("implant config url query parameter names must be 3 or more characters")
ErrDuplicateStageExt = errors.New("stager extension is already used in another C2 profile")
ErrDuplicateStartSessionExt = errors.New("start session extension is already used in another C2 profile")
ErrDuplicateC2ProfileName = errors.New("C2 Profile name is already in use")
ErrUserAgentIllegalCharacters = errors.New("user agent cannot contain the ` character")
+201 -7
View File
@@ -27,6 +27,7 @@ import (
"crypto/sha256"
"encoding/hex"
"errors"
"fmt"
"os"
"path/filepath"
"strings"
@@ -258,7 +259,7 @@ func ImplantProfileByName(name string) (*clientpb.ImplantProfile, error) {
return nil, err
}
err = Session().Where(models.ImplantConfig{
ImplantProfileID: profile.ID,
ImplantProfileID: &profile.ID,
}).First(&config).Error
if err != nil {
return nil, err
@@ -332,6 +333,31 @@ func SearchStageExtensions(stagerExtension string, profileName string) error {
return nil
}
// used to prevent duplicate start session extensions
func SearchStartSessionExtensions(StartSessionFileExt string, profileName string) error {
c2Config := models.HttpC2ImplantConfig{}
err := Session().Where(&models.HttpC2ImplantConfig{
StartSessionFileExtension: StartSessionFileExt,
}).Find(&c2Config).Error
if err != nil {
return err
}
if c2Config.StartSessionFileExtension != "" && profileName != "" {
httpC2Config := models.HttpC2Config{}
err = Session().Where(&models.HttpC2Config{ID: c2Config.HttpC2ConfigID}).Find(&httpC2Config).Error
if err != nil {
return err
}
if httpC2Config.Name == profileName {
return nil
}
return configs.ErrDuplicateStartSessionExt
}
return nil
}
func LoadHTTPC2ConfigByName(name string) (*clientpb.HTTPC2Config, error) {
if len(name) < 1 {
return nil, ErrRecordNotFound
@@ -367,7 +393,7 @@ func LoadHTTPC2ConfigByName(name string) (*clientpb.HTTPC2Config, error) {
// load headers
c2ImplantHeaders := []models.HttpC2Header{}
err = Session().Where(&models.HttpC2Header{
HttpC2ImplantConfigID: c2ImplantConfig.ID,
HttpC2ImplantConfigID: &c2ImplantConfig.ID,
}).Find(&c2ImplantHeaders).Error
if err != nil {
return nil, err
@@ -397,7 +423,7 @@ func LoadHTTPC2ConfigByName(name string) (*clientpb.HTTPC2Config, error) {
// load headers
c2ServerHeaders := []models.HttpC2Header{}
err = Session().Where(&models.HttpC2Header{
HttpC2ServerConfigID: c2ServerConfig.ID,
HttpC2ServerConfigID: &c2ServerConfig.ID,
}).Find(&c2ServerHeaders).Error
if err != nil {
return nil, err
@@ -436,20 +462,74 @@ func SaveHTTPC2Config(httpC2Config *clientpb.HTTPC2Config) error {
func HTTPC2ConfigUpdate(newConf *clientpb.HTTPC2Config, oldConf *clientpb.HTTPC2Config) error {
clientID, _ := uuid.FromString(oldConf.ImplantConfig.ID)
c2Config := models.HTTPC2ConfigFromProtobuf(newConf)
err := Session().Where(&models.ImplantConfig{
err := Session().Where(&models.HttpC2PathSegment{
HttpC2ImplantConfigID: clientID,
}).Delete(&models.HttpC2PathSegment{})
if err.Error != nil {
return err.Error
}
err = Session().Where(&models.ImplantConfig{
ID: clientID,
}).Updates(c2Config.ImplantConfig)
if err != nil {
if err.Error != nil {
return err.Error
}
for _, segment := range c2Config.ImplantConfig.PathSegments {
segment.HttpC2ImplantConfigID = clientID
err = Session().Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&segment)
if err.Error != nil {
return err.Error
}
}
serverID, _ := uuid.FromString(oldConf.ServerConfig.ID)
err = Session().Where(&models.HttpC2Cookie{
HttpC2ServerConfigID: serverID,
}).Delete(&models.HttpC2Cookie{})
if err.Error != nil {
return err.Error
}
err = Session().Where(&models.HttpC2Header{
HttpC2ServerConfigID: &serverID,
}).Delete(&models.HttpC2Header{})
if err.Error != nil {
return err.Error
}
serverID, _ := uuid.FromString(oldConf.ImplantConfig.ID)
err = Session().Where(&models.HttpC2ServerConfig{
ID: serverID,
}).Updates(c2Config.ServerConfig)
if err != nil {
if err.Error != nil {
return err.Error
}
for _, cookie := range c2Config.ServerConfig.Cookies {
cookie.HttpC2ServerConfigID = serverID
err = Session().Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&cookie)
if err.Error != nil {
return err.Error
}
}
for _, header := range c2Config.ServerConfig.Headers {
header.HttpC2ServerConfigID = &serverID
err = Session().Clauses(clause.OnConflict{
UpdateAll: true,
}).Create(&header)
if err.Error != nil {
return err.Error
}
}
return nil
}
@@ -537,6 +617,35 @@ func ListenerJobs() ([]*clientpb.ListenerJob, error) {
}
func DeleteListener(JobID uint32) error {
// Determine which type of listener this is and delete its record from the corresponding table
var deleteErr error
// JobID is unique so if the JobID exists, it is the only record in the table with that JobID
listener := &models.ListenerJob{}
result := Session().Where(&models.ListenerJob{JobID: JobID}).First(&listener)
if result.Error != nil {
return result.Error
}
if listener == nil {
return fmt.Errorf("Job ID %d not found in database", JobID)
}
listenerID := listener.ID
switch listener.Type {
case constants.HttpStr, constants.HttpsStr:
deleteErr = Session().Where(&models.HTTPListener{ListenerJobID: listenerID}).Delete(&models.HTTPListener{}).Error
case constants.DnsStr:
deleteErr = Session().Where(&models.DNSListener{ListenerJobID: listenerID}).Delete(&models.DNSListener{}).Error
case constants.MtlsStr:
deleteErr = Session().Where(&models.MtlsListener{ListenerJobID: listenerID}).Delete(&models.MtlsListener{}).Error
case constants.WGStr:
deleteErr = Session().Where(&models.WGListener{ListenerJobID: listenerID}).Delete(&models.WGListener{}).Error
case constants.MultiplayerModeStr:
deleteErr = Session().Where(&models.MultiplayerListener{ListenerJobID: listenerID}).Delete(&models.MultiplayerListener{}).Error
}
if deleteErr != nil {
return deleteErr
}
return Session().Where(&models.ListenerJob{JobID: JobID}).Delete(&models.ListenerJob{}).Error
}
@@ -587,6 +696,9 @@ func DeleteProfile(name string) error {
uuid, _ := uuid.FromString(profile.Config.ID)
// delete linked ImplantC2
err = Session().Where(&models.ImplantC2{ImplantConfigID: uuid}).Delete(&models.ImplantC2{}).Error
// delete linked ImplantConfig
err = Session().Where(&models.ImplantConfig{ID: uuid}).Delete(&models.ImplantConfig{}).Error
@@ -1304,3 +1416,85 @@ func SaveResourceID(r *clientpb.ResourceID) error {
}
return nil
}
// Certificates
func GetCertificateInfo(categoryOptions uint32, cn string) ([]*models.Certificate, error) {
// Enumerate the options for transport and role
// First, defining the transport filters
const (
MTLSTransport uint32 = 1 << iota
HTTPSTransport
AllTransports
)
// Now, defining the role filters
const (
ServerRole uint32 = 8 << iota
ImplantRole
AllRoles
)
certInfo := []*models.Certificate{}
dbSession := Session()
var err error
switch categoryOptions {
case MTLSTransport | ServerRole:
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn, CAType: "mtls-server"}).Find(&certInfo).Error
} else {
err = dbSession.Where(&models.Certificate{CAType: "mtls-server"}).Find(&certInfo).Error
}
case (MTLSTransport | ImplantRole), (AllTransports | ImplantRole):
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn, CAType: "mtls-implant"}).Find(&certInfo).Error
} else {
err = dbSession.Where(&models.Certificate{CAType: "mtls-implant"}).Find(&certInfo).Error
}
case MTLSTransport | AllRoles:
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn}).Where(
dbSession.Where(&models.Certificate{CAType: "mtls-server"}).Or(
&models.Certificate{CAType: "mtls-implant"},
),
).Find(&certInfo).Error
} else {
err = dbSession.Where(dbSession.Where(&models.Certificate{CAType: "mtls-server"}).Or(
&models.Certificate{CAType: "mtls-implant"})).Find(&certInfo).Error
}
case HTTPSTransport | ImplantRole:
// This does not currently exist (HTTPS certs are not issued for implants), so return nothing
return certInfo, nil
case (HTTPSTransport | ServerRole), (HTTPSTransport | AllRoles):
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn, CAType: "https"}).Find(&certInfo).Error
} else {
err = dbSession.Where(&models.Certificate{CAType: "https"}).Find(&certInfo).Error
}
case AllTransports | ServerRole:
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn}).Where(
dbSession.Where(&models.Certificate{CAType: "mtls-server"}).Or(
&models.Certificate{CAType: "https"},
),
).Find(&certInfo).Error
} else {
err = dbSession.Where(dbSession.Where(&models.Certificate{CAType: "mtls-server"}).Or(
&models.Certificate{CAType: "https"},
),
).Find(&certInfo).Error
}
case AllRoles | AllTransports:
if cn != "" {
err = dbSession.Where(&models.Certificate{CommonName: cn}).Find(&certInfo).Error
} else {
err = dbSession.Where(&models.Certificate{}).Find(&certInfo).Error
}
}
if err != nil {
return nil, err
}
// Processing of the data can occur at the caller. It does not need to happen here.
return certInfo, nil
}
+1 -1
View File
@@ -29,7 +29,7 @@ import (
// Host - Represents a host machine
type Host struct {
ID uuid.UUID `gorm:"primaryKey;->;<-:create;type:uuid;"`
HostUUID uuid.UUID `gorm:"type:uuid;"`
HostUUID uuid.UUID `gorm:"type:uuid;unique"`
CreatedAt time.Time `gorm:"->;<-:create;"`
Hostname string
+3 -3
View File
@@ -203,9 +203,9 @@ func (h *HttpC2Cookie) ToProtobuf() *clientpb.HTTPC2Cookie {
// HttpC2Header - HTTP C2 Header (server and implant)
type HttpC2Header struct {
ID uuid.UUID `gorm:"primaryKey;->;<-:create;type:uuid;"`
HttpC2ServerConfigID uuid.UUID `gorm:"type:uuid;"`
HttpC2ImplantConfigID uuid.UUID `gorm:"type:uuid;"`
ID uuid.UUID `gorm:"primaryKey;->;<-:create;type:uuid;"`
HttpC2ServerConfigID *uuid.UUID `gorm:"type:uuid;"`
HttpC2ImplantConfigID *uuid.UUID `gorm:"type:uuid;"`
Method string
Name string
+13 -5
View File
@@ -144,7 +144,7 @@ func ImplantBuildFromProtobuf(ib *clientpb.ImplantBuild) *ImplantBuild {
// ImplantConfig - An implant build configuration
type ImplantConfig struct {
ID uuid.UUID `gorm:"primaryKey;->;<-:create;type:uuid;"`
ImplantProfileID uuid.UUID
ImplantProfileID *uuid.UUID
ImplantBuilds []ImplantBuild
CreatedAt time.Time `gorm:"->;<-:create;"`
@@ -237,9 +237,8 @@ func (ic *ImplantConfig) ToProtobuf() *clientpb.ImplantConfig {
implantBuilds = append(implantBuilds, implantBuild.ToProtobuf())
}
config := &clientpb.ImplantConfig{
ID: ic.ID.String(),
ImplantBuilds: implantBuilds,
ImplantProfileID: ic.ImplantProfileID.String(),
ID: ic.ID.String(),
ImplantBuilds: implantBuilds,
IsBeacon: ic.IsBeacon,
BeaconInterval: ic.BeaconInterval,
@@ -286,6 +285,11 @@ func (ic *ImplantConfig) ToProtobuf() *clientpb.ImplantConfig {
IncludeWG: ic.IncludeWG,
IncludeTCP: ic.IncludeTCP,
}
if ic.ImplantProfileID != nil {
config.ImplantProfileID = ic.ImplantProfileID.String()
}
// Copy Canary Domains
config.CanaryDomains = []string{}
for _, canaryDomain := range ic.CanaryDomains {
@@ -432,7 +436,11 @@ func ImplantConfigFromProtobuf(pbConfig *clientpb.ImplantConfig) *ImplantConfig
cfg.ID = id
cfg.ImplantBuilds = implantBuilds
profileID, _ := uuid.FromString(pbConfig.ImplantProfileID)
cfg.ImplantProfileID = profileID
if profileID == uuid.Nil {
cfg.ImplantProfileID = nil
} else {
cfg.ImplantProfileID = &profileID
}
cfg.IsBeacon = pbConfig.IsBeacon
cfg.BeaconInterval = pbConfig.BeaconInterval
+21 -14
View File
@@ -50,7 +50,14 @@ func newDBClient() *gorm.DB {
panic(fmt.Sprintf("Unknown DB Dialect: '%s'", dbConfig.Dialect))
}
err := dbClient.AutoMigrate(
// We cannot pass all of these into AutoMigrate at once because if one fails, subsequent models will not be created
var allDBModels []interface{} = append(make([]interface{}, 0),
&models.HttpC2Header{},
&models.HttpC2ServerConfig{},
&models.HttpC2ImplantConfig{},
&models.HttpC2Config{},
&models.HttpC2URLParameter{},
&models.HttpC2PathSegment{},
&models.Beacon{},
&models.BeaconTask{},
&models.DNSCanary{},
@@ -62,30 +69,25 @@ func newDBClient() *gorm.DB {
&models.CrackFileChunk{},
&models.Certificate{},
&models.Host{},
&models.HttpC2Config{},
&models.HttpC2ImplantConfig{},
&models.HttpC2ServerConfig{},
&models.HttpC2Header{},
&models.HttpC2PathSegment{},
&models.KeyValue{},
&models.WGKeys{},
&models.WGPeer{},
&models.ResourceID{},
&models.HttpC2Cookie{},
&models.HttpC2URLParameter{},
&models.IOC{},
&models.ExtensionData{},
&models.ImplantBuild{},
&models.ImplantProfile{},
&models.ImplantConfig{},
&models.ImplantBuild{},
&models.ImplantC2{},
&models.EncoderAsset{},
&models.KeyExHistory{},
&models.KeyValue{},
&models.CanaryDomain{},
&models.Loot{},
&models.Credential{},
&models.Operator{},
&models.Website{},
&models.WebContent{},
&models.WGKeys{},
&models.WGPeer{},
&models.ListenerJob{},
&models.HTTPListener{},
&models.DNSListener{},
@@ -94,10 +96,15 @@ func newDBClient() *gorm.DB {
&models.MtlsListener{},
&models.DnsDomain{},
&models.MonitoringProvider{},
&models.ResourceID{},
)
if err != nil {
clientLog.Error(err)
var err error
for _, model := range allDBModels {
err = dbClient.AutoMigrate(model)
if err != nil {
clientLog.Error(err)
}
}
// Get generic database object sql.DB to use its functions
+5 -1
View File
@@ -77,7 +77,11 @@ func ImplantConfigSave(config *clientpb.ImplantConfig) (*clientpb.ImplantConfig,
} else {
id, _ := uuid.FromString(dbConfig.ImplantProfileID)
modelConfig.ImplantProfileID = id
if id == uuid.Nil {
modelConfig.ImplantProfileID = nil
} else {
modelConfig.ImplantProfileID = &id
}
// this avoids gorm saving duplicate c2 objects ...
tempC2 := modelConfig.C2
+5 -1
View File
@@ -54,7 +54,11 @@ func SaveImplantProfile(pbProfile *clientpb.ImplantProfile) (*clientpb.ImplantPr
profile.ImplantConfig.ID = configID
profileID, _ := uuid.FromString(dbProfile.ID)
profile.ImplantConfig.ImplantProfileID = profileID
if profileID == uuid.Nil {
profile.ImplantConfig.ImplantProfileID = nil
} else {
profile.ImplantConfig.ImplantProfileID = &profileID
}
for _, c2 := range dbProfile.Config.C2 {
id, _ := uuid.FromString(c2.ID)
+5
View File
@@ -68,6 +68,11 @@ func (rpc *Server) SaveHTTPC2Profile(ctx context.Context, req *clientpb.HTTPC2Co
return nil, err
}
err = db.SearchStartSessionExtensions(req.C2Config.ImplantConfig.StartSessionFileExtension, profileName)
if err != nil {
return nil, err
}
httpC2Config, err := db.LoadHTTPC2ConfigByName(req.C2Config.Name)
if err != nil {
return nil, err
+102
View File
@@ -0,0 +1,102 @@
package rpc
import (
"context"
"crypto/x509"
"encoding/pem"
"strings"
"github.com/bishopfox/sliver/protobuf/clientpb"
"github.com/bishopfox/sliver/server/certs"
"github.com/bishopfox/sliver/server/db"
"github.com/bishopfox/sliver/server/db/models"
)
/*
Sliver Implant Framework
Copyright (C) 2024 Bishop Fox
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
const (
timeFormat = "2006-01-02 15:04:05 UTC-0700"
)
func convertDatabaseRecordToProtobuf(record *models.Certificate) *clientpb.CertificateData {
if record == nil {
return nil
}
certData := &clientpb.CertificateData{}
// These values are default until we parse the cert
certData.ValidityStart = "Unknown (could not parse certificate)"
certData.ValidityExpiry = "Unknown (could not parse certificate)"
certData.CN = record.CommonName
certData.ID = record.ID.String()
certData.CreationTime = record.CreatedAt.Format(timeFormat)
switch record.CAType {
case certs.MtlsImplantCA:
certData.Type = "MTLS (Implant)"
case certs.MtlsServerCA:
certData.Type = "MTLS (Server)"
case certs.HTTPSCA:
certData.Type = "HTTPS"
default:
certData.Type = record.CAType
}
switch record.KeyType {
case certs.ECCKey:
certData.KeyAlgorithm = "ECC"
case certs.RSAKey:
certData.KeyAlgorithm = "RSA"
default:
certData.KeyAlgorithm = strings.ToUpper(record.KeyType)
}
// To get the validity period, we need to parse the certificate information
pemBlock, _ := pem.Decode([]byte(record.CertificatePEM))
if pemBlock == nil {
return certData
}
certificate, err := x509.ParseCertificate(pemBlock.Bytes)
if err != nil {
return certData
}
certData.ValidityStart = certificate.NotBefore.Format(timeFormat)
certData.ValidityExpiry = certificate.NotAfter.Format(timeFormat)
return certData
}
func (rpc *Server) GetCertificateInfo(ctx context.Context, req *clientpb.CertificatesReq) (*clientpb.CertificateInfo, error) {
certInfo := clientpb.CertificateInfo{}
certInfoDB, err := db.GetCertificateInfo(req.CategoryFilters, req.CN)
if err != nil {
return nil, err
}
certInfo.Info = make([]*clientpb.CertificateData, len(certInfoDB))
for idx, record := range certInfoDB {
certInfo.Info[idx] = convertDatabaseRecordToProtobuf(record)
}
return &certInfo, nil
}
+14 -3
View File
@@ -417,10 +417,21 @@ func (rpc *Server) GenerateExternalGetBuildConfig(ctx context.Context, req *clie
return nil, status.Error(codes.Internal, fmt.Sprintf("Unable to load HTTP C2 Configuration: %s", err))
}
encoders := map[string]uint64{
"base64": encoders.Base64EncoderID,
"base58": encoders.Base58EncoderID,
"base32": encoders.Base32EncoderID,
"hex": encoders.HexEncoderID,
"english": encoders.EnglishEncoderID,
"gzip": encoders.GzipEncoderID,
"png": encoders.PNGEncoderID,
}
return &clientpb.ExternalImplantConfig{
Config: implantConfig,
Build: build,
HTTPC2: httpC2Config,
Config: implantConfig,
Build: build,
HTTPC2: httpC2Config,
Encoders: encoders,
}, nil
}
+12 -2
View File
@@ -1,3 +1,4 @@
//go:build windows
// +build windows
package pb
@@ -5,6 +6,7 @@ package pb
import (
"fmt"
"log"
"strings"
"github.com/cheggaaa/pb/v3/termutil"
)
@@ -24,17 +26,25 @@ func (p *Pool) print(first bool) bool {
}
coords.X = 0
err = termutil.SetCursorPos(coords)
err = termutil.SetCursorPos(coords)
if err != nil {
log.Panic(err)
}
}
cols, err := termutil.TerminalWidth()
if err != nil {
cols = defaultBarWidth
}
isFinished := true
for _, bar := range p.bars {
if !bar.IsFinished() {
isFinished = false
}
out += fmt.Sprintf("\r%s\n", bar.String())
result := bar.String()
if r := cols - CellCount(result); r > 0 {
result += strings.Repeat(" ", r)
}
out += fmt.Sprintf("\r%s\n", result)
}
if p.Output != nil {
fmt.Fprint(p.Output, out)
+7 -1
View File
@@ -1,3 +1,4 @@
//go:build linux || darwin || freebsd || netbsd || openbsd || solaris || dragonfly || plan9 || aix
// +build linux darwin freebsd netbsd openbsd solaris dragonfly plan9 aix
package pb
@@ -5,6 +6,7 @@ package pb
import (
"fmt"
"os"
"strings"
"github.com/cheggaaa/pb/v3/termutil"
)
@@ -31,7 +33,11 @@ func (p *Pool) print(first bool) bool {
isFinished = false
}
bar.SetWidth(cols)
out += fmt.Sprintf("\r%s\n", bar.String())
result := bar.String()
if r := cols - CellCount(result); r > 0 {
result += strings.Repeat(" ", r)
}
out += fmt.Sprintf("\r%s\n", result)
}
if p.Output != nil {
fmt.Fprint(p.Output, out)
+1 -2
View File
@@ -138,9 +138,8 @@ func lockEcho() (err error) {
}
newState := oldState
const ENABLE_ECHO_INPUT = 0x0004
const ENABLE_LINE_INPUT = 0x0002
newState = newState & (^(ENABLE_LINE_INPUT | ENABLE_ECHO_INPUT))
newState = newState & (^ENABLE_LINE_INPUT)
if _, _, e := syscall.Syscall(setConsoleMode.Addr(), 2, uintptr(syscall.Stdout), uintptr(newState), 0); e != 0 {
err = fmt.Errorf("Can't set terminal settings: %v", e)
return
+35 -9
View File
@@ -45,15 +45,21 @@ func (e *Error) Error() string {
return fmt.Sprintf("running %v: exit status %v: %v", e.cmd.Args, e.ExitStatus(), e.msg)
}
var isNotExistPatterns = []string{
"Bad rule (does a matching rule exist in that chain?).\n",
"No chain/target/match by that name.\n",
"No such file or directory",
"does not exist",
}
// IsNotExist returns true if the error is due to the chain or rule not existing
func (e *Error) IsNotExist() bool {
if e.ExitStatus() != 1 {
return false
for _, str := range isNotExistPatterns {
if strings.Contains(e.msg, str) {
return true
}
}
msgNoRuleExist := "Bad rule (does a matching rule exist in that chain?).\n"
msgNoChainExist := "No chain/target/match by that name.\n"
msgENOENT := "No such file or directory"
return strings.Contains(e.msg, msgNoRuleExist) || strings.Contains(e.msg, msgNoChainExist) || strings.Contains(e.msg, msgENOENT)
return false
}
// Protocol to differentiate between IPv4 and IPv6
@@ -106,8 +112,20 @@ func Timeout(timeout int) option {
}
}
// New creates a new IPTables configured with the options passed as parameter.
// For backwards compatibility, by default always uses IPv4 and timeout 0.
func Path(path string) option {
return func(ipt *IPTables) {
ipt.path = path
}
}
// New creates a new IPTables configured with the options passed as parameters.
// Supported parameters are:
//
// IPFamily(Protocol)
// Timeout(int)
// Path(string)
//
// For backwards compatibility, by default New uses IPv4 and timeout 0.
// i.e. you can create an IPv6 IPTables using a timeout of 5 seconds passing
// the IPFamily and Timeout options as follow:
//
@@ -117,13 +135,21 @@ func New(opts ...option) (*IPTables, error) {
ipt := &IPTables{
proto: ProtocolIPv4,
timeout: 0,
path: "",
}
for _, opt := range opts {
opt(ipt)
}
path, err := exec.LookPath(getIptablesCommand(ipt.proto))
// if path wasn't preset through New(Path()), autodiscover it
cmd := ""
if ipt.path == "" {
cmd = getIptablesCommand(ipt.proto)
} else {
cmd = ipt.path
}
path, err := exec.LookPath(cmd)
if err != nil {
return nil, err
}
-191
View File
@@ -1,191 +0,0 @@
Apache License
Version 2.0, January 2004
http://www.apache.org/licenses/
TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
1. Definitions.
"License" shall mean the terms and conditions for use, reproduction, and
distribution as defined by Sections 1 through 9 of this document.
"Licensor" shall mean the copyright owner or entity authorized by the copyright
owner that is granting the License.
"Legal Entity" shall mean the union of the acting entity and all other entities
that control, are controlled by, or are under common control with that entity.
For the purposes of this definition, "control" means (i) the power, direct or
indirect, to cause the direction or management of such entity, whether by
contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the
outstanding shares, or (iii) beneficial ownership of such entity.
"You" (or "Your") shall mean an individual or Legal Entity exercising
permissions granted by this License.
"Source" form shall mean the preferred form for making modifications, including
but not limited to software source code, documentation source, and configuration
files.
"Object" form shall mean any form resulting from mechanical transformation or
translation of a Source form, including but not limited to compiled object code,
generated documentation, and conversions to other media types.
"Work" shall mean the work of authorship, whether in Source or Object form, made
available under the License, as indicated by a copyright notice that is included
in or attached to the work (an example is provided in the Appendix below).
"Derivative Works" shall mean any work, whether in Source or Object form, that
is based on (or derived from) the Work and for which the editorial revisions,
annotations, elaborations, or other modifications represent, as a whole, an
original work of authorship. For the purposes of this License, Derivative Works
shall not include works that remain separable from, or merely link (or bind by
name) to the interfaces of, the Work and Derivative Works thereof.
"Contribution" shall mean any work of authorship, including the original version
of the Work and any modifications or additions to that Work or Derivative Works
thereof, that is intentionally submitted to Licensor for inclusion in the Work
by the copyright owner or by an individual or Legal Entity authorized to submit
on behalf of the copyright owner. For the purposes of this definition,
"submitted" means any form of electronic, verbal, or written communication sent
to the Licensor or its representatives, including but not limited to
communication on electronic mailing lists, source code control systems, and
issue tracking systems that are managed by, or on behalf of, the Licensor for
the purpose of discussing and improving the Work, but excluding communication
that is conspicuously marked or otherwise designated in writing by the copyright
owner as "Not a Contribution."
"Contributor" shall mean Licensor and any individual or Legal Entity on behalf
of whom a Contribution has been received by Licensor and subsequently
incorporated within the Work.
2. Grant of Copyright License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable copyright license to reproduce, prepare Derivative Works of,
publicly display, publicly perform, sublicense, and distribute the Work and such
Derivative Works in Source or Object form.
3. Grant of Patent License.
Subject to the terms and conditions of this License, each Contributor hereby
grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
irrevocable (except as stated in this section) patent license to make, have
made, use, offer to sell, sell, import, and otherwise transfer the Work, where
such license applies only to those patent claims licensable by such Contributor
that are necessarily infringed by their Contribution(s) alone or by combination
of their Contribution(s) with the Work to which such Contribution(s) was
submitted. If You institute patent litigation against any entity (including a
cross-claim or counterclaim in a lawsuit) alleging that the Work or a
Contribution incorporated within the Work constitutes direct or contributory
patent infringement, then any patent licenses granted to You under this License
for that Work shall terminate as of the date such litigation is filed.
4. Redistribution.
You may reproduce and distribute copies of the Work or Derivative Works thereof
in any medium, with or without modifications, and in Source or Object form,
provided that You meet the following conditions:
You must give any other recipients of the Work or Derivative Works a copy of
this License; and
You must cause any modified files to carry prominent notices stating that You
changed the files; and
You must retain, in the Source form of any Derivative Works that You distribute,
all copyright, patent, trademark, and attribution notices from the Source form
of the Work, excluding those notices that do not pertain to any part of the
Derivative Works; and
If the Work includes a "NOTICE" text file as part of its distribution, then any
Derivative Works that You distribute must include a readable copy of the
attribution notices contained within such NOTICE file, excluding those notices
that do not pertain to any part of the Derivative Works, in at least one of the
following places: within a NOTICE text file distributed as part of the
Derivative Works; within the Source form or documentation, if provided along
with the Derivative Works; or, within a display generated by the Derivative
Works, if and wherever such third-party notices normally appear. The contents of
the NOTICE file are for informational purposes only and do not modify the
License. You may add Your own attribution notices within Derivative Works that
You distribute, alongside or as an addendum to the NOTICE text from the Work,
provided that such additional attribution notices cannot be construed as
modifying the License.
You may add Your own copyright statement to Your modifications and may provide
additional or different license terms and conditions for use, reproduction, or
distribution of Your modifications, or for any such Derivative Works as a whole,
provided Your use, reproduction, and distribution of the Work otherwise complies
with the conditions stated in this License.
5. Submission of Contributions.
Unless You explicitly state otherwise, any Contribution intentionally submitted
for inclusion in the Work by You to the Licensor shall be under the terms and
conditions of this License, without any additional terms or conditions.
Notwithstanding the above, nothing herein shall supersede or modify the terms of
any separate license agreement you may have executed with Licensor regarding
such Contributions.
6. Trademarks.
This License does not grant permission to use the trade names, trademarks,
service marks, or product names of the Licensor, except as required for
reasonable and customary use in describing the origin of the Work and
reproducing the content of the NOTICE file.
7. Disclaimer of Warranty.
Unless required by applicable law or agreed to in writing, Licensor provides the
Work (and each Contributor provides its Contributions) on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied,
including, without limitation, any warranties or conditions of TITLE,
NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are
solely responsible for determining the appropriateness of using or
redistributing the Work and assume any risks associated with Your exercise of
permissions under this License.
8. Limitation of Liability.
In no event and under no legal theory, whether in tort (including negligence),
contract, or otherwise, unless required by applicable law (such as deliberate
and grossly negligent acts) or agreed to in writing, shall any Contributor be
liable to You for damages, including any direct, indirect, special, incidental,
or consequential damages of any character arising as a result of this License or
out of the use or inability to use the Work (including but not limited to
damages for loss of goodwill, work stoppage, computer failure or malfunction, or
any and all other commercial damages or losses), even if such Contributor has
been advised of the possibility of such damages.
9. Accepting Warranty or Additional Liability.
While redistributing the Work or Derivative Works thereof, You may choose to
offer, and charge a fee for, acceptance of support, warranty, indemnity, or
other liability obligations and/or rights consistent with this License. However,
in accepting such obligations, You may act only on Your own behalf and on Your
sole responsibility, not on behalf of any other Contributor, and only if You
agree to indemnify, defend, and hold each Contributor harmless for any liability
incurred by, or claims asserted against, such Contributor by reason of your
accepting any such warranty or additional liability.
END OF TERMS AND CONDITIONS
APPENDIX: How to apply the Apache License to your work
To apply the Apache License to your work, attach the following boilerplate
notice, with the fields enclosed by brackets "[]" replaced with your own
identifying information. (Don't include the brackets!) The text should be
enclosed in the appropriate comment syntax for the file format. We also
recommend that a file or class name and description of purpose be included on
the same "printed page" as the copyright notice for easier identification within
third-party archives.
Copyright [yyyy] [name of copyright owner]
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
-5
View File
@@ -1,5 +0,0 @@
CoreOS Project
Copyright 2018 CoreOS, Inc
This product includes software developed at CoreOS, Inc.
(http://www.coreos.com/).
-266
View File
@@ -1,266 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Integration with the systemd D-Bus API. See http://www.freedesktop.org/wiki/Software/systemd/dbus/
package dbus
import (
"context"
"encoding/hex"
"fmt"
"os"
"strconv"
"strings"
"sync"
"github.com/godbus/dbus/v5"
)
const (
alpha = `abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ`
num = `0123456789`
alphanum = alpha + num
signalBuffer = 100
)
// needsEscape checks whether a byte in a potential dbus ObjectPath needs to be escaped
func needsEscape(i int, b byte) bool {
// Escape everything that is not a-z-A-Z-0-9
// Also escape 0-9 if it's the first character
return strings.IndexByte(alphanum, b) == -1 ||
(i == 0 && strings.IndexByte(num, b) != -1)
}
// PathBusEscape sanitizes a constituent string of a dbus ObjectPath using the
// rules that systemd uses for serializing special characters.
func PathBusEscape(path string) string {
// Special case the empty string
if len(path) == 0 {
return "_"
}
n := []byte{}
for i := 0; i < len(path); i++ {
c := path[i]
if needsEscape(i, c) {
e := fmt.Sprintf("_%x", c)
n = append(n, []byte(e)...)
} else {
n = append(n, c)
}
}
return string(n)
}
// pathBusUnescape is the inverse of PathBusEscape.
func pathBusUnescape(path string) string {
if path == "_" {
return ""
}
n := []byte{}
for i := 0; i < len(path); i++ {
c := path[i]
if c == '_' && i+2 < len(path) {
res, err := hex.DecodeString(path[i+1 : i+3])
if err == nil {
n = append(n, res...)
}
i += 2
} else {
n = append(n, c)
}
}
return string(n)
}
// Conn is a connection to systemd's dbus endpoint.
type Conn struct {
// sysconn/sysobj are only used to call dbus methods
sysconn *dbus.Conn
sysobj dbus.BusObject
// sigconn/sigobj are only used to receive dbus signals
sigconn *dbus.Conn
sigobj dbus.BusObject
jobListener struct {
jobs map[dbus.ObjectPath]chan<- string
sync.Mutex
}
subStateSubscriber struct {
updateCh chan<- *SubStateUpdate
errCh chan<- error
sync.Mutex
ignore map[dbus.ObjectPath]int64
cleanIgnore int64
}
propertiesSubscriber struct {
updateCh chan<- *PropertiesUpdate
errCh chan<- error
sync.Mutex
}
}
// Deprecated: use NewWithContext instead.
func New() (*Conn, error) {
return NewWithContext(context.Background())
}
// NewWithContext establishes a connection to any available bus and authenticates.
// Callers should call Close() when done with the connection.
func NewWithContext(ctx context.Context) (*Conn, error) {
conn, err := NewSystemConnectionContext(ctx)
if err != nil && os.Geteuid() == 0 {
return NewSystemdConnectionContext(ctx)
}
return conn, err
}
// Deprecated: use NewSystemConnectionContext instead.
func NewSystemConnection() (*Conn, error) {
return NewSystemConnectionContext(context.Background())
}
// NewSystemConnectionContext establishes a connection to the system bus and authenticates.
// Callers should call Close() when done with the connection.
func NewSystemConnectionContext(ctx context.Context) (*Conn, error) {
return NewConnection(func() (*dbus.Conn, error) {
return dbusAuthHelloConnection(ctx, dbus.SystemBusPrivate)
})
}
// Deprecated: use NewUserConnectionContext instead.
func NewUserConnection() (*Conn, error) {
return NewUserConnectionContext(context.Background())
}
// NewUserConnectionContext establishes a connection to the session bus and
// authenticates. This can be used to connect to systemd user instances.
// Callers should call Close() when done with the connection.
func NewUserConnectionContext(ctx context.Context) (*Conn, error) {
return NewConnection(func() (*dbus.Conn, error) {
return dbusAuthHelloConnection(ctx, dbus.SessionBusPrivate)
})
}
// Deprecated: use NewSystemdConnectionContext instead.
func NewSystemdConnection() (*Conn, error) {
return NewSystemdConnectionContext(context.Background())
}
// NewSystemdConnectionContext establishes a private, direct connection to systemd.
// This can be used for communicating with systemd without a dbus daemon.
// Callers should call Close() when done with the connection.
func NewSystemdConnectionContext(ctx context.Context) (*Conn, error) {
return NewConnection(func() (*dbus.Conn, error) {
// We skip Hello when talking directly to systemd.
return dbusAuthConnection(ctx, func(opts ...dbus.ConnOption) (*dbus.Conn, error) {
return dbus.Dial("unix:path=/run/systemd/private", opts...)
})
})
}
// Close closes an established connection.
func (c *Conn) Close() {
c.sysconn.Close()
c.sigconn.Close()
}
// Connected returns whether conn is connected
func (c *Conn) Connected() bool {
return c.sysconn.Connected() && c.sigconn.Connected()
}
// NewConnection establishes a connection to a bus using a caller-supplied function.
// This allows connecting to remote buses through a user-supplied mechanism.
// The supplied function may be called multiple times, and should return independent connections.
// The returned connection must be fully initialised: the org.freedesktop.DBus.Hello call must have succeeded,
// and any authentication should be handled by the function.
func NewConnection(dialBus func() (*dbus.Conn, error)) (*Conn, error) {
sysconn, err := dialBus()
if err != nil {
return nil, err
}
sigconn, err := dialBus()
if err != nil {
sysconn.Close()
return nil, err
}
c := &Conn{
sysconn: sysconn,
sysobj: systemdObject(sysconn),
sigconn: sigconn,
sigobj: systemdObject(sigconn),
}
c.subStateSubscriber.ignore = make(map[dbus.ObjectPath]int64)
c.jobListener.jobs = make(map[dbus.ObjectPath]chan<- string)
// Setup the listeners on jobs so that we can get completions
c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
"type='signal', interface='org.freedesktop.systemd1.Manager', member='JobRemoved'")
c.dispatch()
return c, nil
}
// GetManagerProperty returns the value of a property on the org.freedesktop.systemd1.Manager
// interface. The value is returned in its string representation, as defined at
// https://developer.gnome.org/glib/unstable/gvariant-text.html.
func (c *Conn) GetManagerProperty(prop string) (string, error) {
variant, err := c.sysobj.GetProperty("org.freedesktop.systemd1.Manager." + prop)
if err != nil {
return "", err
}
return variant.String(), nil
}
func dbusAuthConnection(ctx context.Context, createBus func(opts ...dbus.ConnOption) (*dbus.Conn, error)) (*dbus.Conn, error) {
conn, err := createBus(dbus.WithContext(ctx))
if err != nil {
return nil, err
}
// Only use EXTERNAL method, and hardcode the uid (not username)
// to avoid a username lookup (which requires a dynamically linked
// libc)
methods := []dbus.Auth{dbus.AuthExternal(strconv.Itoa(os.Getuid()))}
err = conn.Auth(methods)
if err != nil {
conn.Close()
return nil, err
}
return conn, nil
}
func dbusAuthHelloConnection(ctx context.Context, createBus func(opts ...dbus.ConnOption) (*dbus.Conn, error)) (*dbus.Conn, error) {
conn, err := dbusAuthConnection(ctx, createBus)
if err != nil {
return nil, err
}
if err = conn.Hello(); err != nil {
conn.Close()
return nil, err
}
return conn, nil
}
func systemdObject(conn *dbus.Conn) dbus.BusObject {
return conn.Object("org.freedesktop.systemd1", dbus.ObjectPath("/org/freedesktop/systemd1"))
}
-864
View File
@@ -1,864 +0,0 @@
// Copyright 2015, 2018 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbus
import (
"context"
"errors"
"fmt"
"path"
"strconv"
"github.com/godbus/dbus/v5"
)
// Who can be used to specify which process to kill in the unit via the KillUnitWithTarget API
type Who string
const (
// All sends the signal to all processes in the unit
All Who = "all"
// Main sends the signal to the main process of the unit
Main Who = "main"
// Control sends the signal to the control process of the unit
Control Who = "control"
)
func (c *Conn) jobComplete(signal *dbus.Signal) {
var id uint32
var job dbus.ObjectPath
var unit string
var result string
dbus.Store(signal.Body, &id, &job, &unit, &result)
c.jobListener.Lock()
out, ok := c.jobListener.jobs[job]
if ok {
out <- result
delete(c.jobListener.jobs, job)
}
c.jobListener.Unlock()
}
func (c *Conn) startJob(ctx context.Context, ch chan<- string, job string, args ...interface{}) (int, error) {
if ch != nil {
c.jobListener.Lock()
defer c.jobListener.Unlock()
}
var p dbus.ObjectPath
err := c.sysobj.CallWithContext(ctx, job, 0, args...).Store(&p)
if err != nil {
return 0, err
}
if ch != nil {
c.jobListener.jobs[p] = ch
}
// ignore error since 0 is fine if conversion fails
jobID, _ := strconv.Atoi(path.Base(string(p)))
return jobID, nil
}
// Deprecated: use StartUnitContext instead.
func (c *Conn) StartUnit(name string, mode string, ch chan<- string) (int, error) {
return c.StartUnitContext(context.Background(), name, mode, ch)
}
// StartUnitContext enqueues a start job and depending jobs, if any (unless otherwise
// specified by the mode string).
//
// Takes the unit to activate, plus a mode string. The mode needs to be one of
// replace, fail, isolate, ignore-dependencies, ignore-requirements. If
// "replace" the call will start the unit and its dependencies, possibly
// replacing already queued jobs that conflict with this. If "fail" the call
// will start the unit and its dependencies, but will fail if this would change
// an already queued job. If "isolate" the call will start the unit in question
// and terminate all units that aren't dependencies of it. If
// "ignore-dependencies" it will start a unit but ignore all its dependencies.
// If "ignore-requirements" it will start a unit but only ignore the
// requirement dependencies. It is not recommended to make use of the latter
// two options.
//
// If the provided channel is non-nil, a result string will be sent to it upon
// job completion: one of done, canceled, timeout, failed, dependency, skipped.
// done indicates successful execution of a job. canceled indicates that a job
// has been canceled before it finished execution. timeout indicates that the
// job timeout was reached. failed indicates that the job failed. dependency
// indicates that a job this job has been depending on failed and the job hence
// has been removed too. skipped indicates that a job was skipped because it
// didn't apply to the units current state.
//
// If no error occurs, the ID of the underlying systemd job will be returned. There
// does exist the possibility for no error to be returned, but for the returned job
// ID to be 0. In this case, the actual underlying ID is not 0 and this datapoint
// should not be considered authoritative.
//
// If an error does occur, it will be returned to the user alongside a job ID of 0.
func (c *Conn) StartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.StartUnit", name, mode)
}
// Deprecated: use StopUnitContext instead.
func (c *Conn) StopUnit(name string, mode string, ch chan<- string) (int, error) {
return c.StopUnitContext(context.Background(), name, mode, ch)
}
// StopUnitContext is similar to StartUnitContext, but stops the specified unit
// rather than starting it.
func (c *Conn) StopUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.StopUnit", name, mode)
}
// Deprecated: use ReloadUnitContext instead.
func (c *Conn) ReloadUnit(name string, mode string, ch chan<- string) (int, error) {
return c.ReloadUnitContext(context.Background(), name, mode, ch)
}
// ReloadUnitContext reloads a unit. Reloading is done only if the unit
// is already running, and fails otherwise.
func (c *Conn) ReloadUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.ReloadUnit", name, mode)
}
// Deprecated: use RestartUnitContext instead.
func (c *Conn) RestartUnit(name string, mode string, ch chan<- string) (int, error) {
return c.RestartUnitContext(context.Background(), name, mode, ch)
}
// RestartUnitContext restarts a service. If a service is restarted that isn't
// running it will be started.
func (c *Conn) RestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.RestartUnit", name, mode)
}
// Deprecated: use TryRestartUnitContext instead.
func (c *Conn) TryRestartUnit(name string, mode string, ch chan<- string) (int, error) {
return c.TryRestartUnitContext(context.Background(), name, mode, ch)
}
// TryRestartUnitContext is like RestartUnitContext, except that a service that
// isn't running is not affected by the restart.
func (c *Conn) TryRestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.TryRestartUnit", name, mode)
}
// Deprecated: use ReloadOrRestartUnitContext instead.
func (c *Conn) ReloadOrRestartUnit(name string, mode string, ch chan<- string) (int, error) {
return c.ReloadOrRestartUnitContext(context.Background(), name, mode, ch)
}
// ReloadOrRestartUnitContext attempts a reload if the unit supports it and use
// a restart otherwise.
func (c *Conn) ReloadOrRestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.ReloadOrRestartUnit", name, mode)
}
// Deprecated: use ReloadOrTryRestartUnitContext instead.
func (c *Conn) ReloadOrTryRestartUnit(name string, mode string, ch chan<- string) (int, error) {
return c.ReloadOrTryRestartUnitContext(context.Background(), name, mode, ch)
}
// ReloadOrTryRestartUnitContext attempts a reload if the unit supports it,
// and use a "Try" flavored restart otherwise.
func (c *Conn) ReloadOrTryRestartUnitContext(ctx context.Context, name string, mode string, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.ReloadOrTryRestartUnit", name, mode)
}
// Deprecated: use StartTransientUnitContext instead.
func (c *Conn) StartTransientUnit(name string, mode string, properties []Property, ch chan<- string) (int, error) {
return c.StartTransientUnitContext(context.Background(), name, mode, properties, ch)
}
// StartTransientUnitContext may be used to create and start a transient unit, which
// will be released as soon as it is not running or referenced anymore or the
// system is rebooted. name is the unit name including suffix, and must be
// unique. mode is the same as in StartUnitContext, properties contains properties
// of the unit.
func (c *Conn) StartTransientUnitContext(ctx context.Context, name string, mode string, properties []Property, ch chan<- string) (int, error) {
return c.startJob(ctx, ch, "org.freedesktop.systemd1.Manager.StartTransientUnit", name, mode, properties, make([]PropertyCollection, 0))
}
// Deprecated: use KillUnitContext instead.
func (c *Conn) KillUnit(name string, signal int32) {
c.KillUnitContext(context.Background(), name, signal)
}
// KillUnitContext takes the unit name and a UNIX signal number to send.
// All of the unit's processes are killed.
func (c *Conn) KillUnitContext(ctx context.Context, name string, signal int32) {
c.KillUnitWithTarget(ctx, name, All, signal)
}
// KillUnitWithTarget is like KillUnitContext, but allows you to specify which
// process in the unit to send the signal to.
func (c *Conn) KillUnitWithTarget(ctx context.Context, name string, target Who, signal int32) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.KillUnit", 0, name, string(target), signal).Store()
}
// Deprecated: use ResetFailedUnitContext instead.
func (c *Conn) ResetFailedUnit(name string) error {
return c.ResetFailedUnitContext(context.Background(), name)
}
// ResetFailedUnitContext resets the "failed" state of a specific unit.
func (c *Conn) ResetFailedUnitContext(ctx context.Context, name string) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ResetFailedUnit", 0, name).Store()
}
// Deprecated: use SystemStateContext instead.
func (c *Conn) SystemState() (*Property, error) {
return c.SystemStateContext(context.Background())
}
// SystemStateContext returns the systemd state. Equivalent to
// systemctl is-system-running.
func (c *Conn) SystemStateContext(ctx context.Context) (*Property, error) {
var err error
var prop dbus.Variant
obj := c.sysconn.Object("org.freedesktop.systemd1", "/org/freedesktop/systemd1")
err = obj.CallWithContext(ctx, "org.freedesktop.DBus.Properties.Get", 0, "org.freedesktop.systemd1.Manager", "SystemState").Store(&prop)
if err != nil {
return nil, err
}
return &Property{Name: "SystemState", Value: prop}, nil
}
// getProperties takes the unit path and returns all of its dbus object properties, for the given dbus interface.
func (c *Conn) getProperties(ctx context.Context, path dbus.ObjectPath, dbusInterface string) (map[string]interface{}, error) {
var err error
var props map[string]dbus.Variant
if !path.IsValid() {
return nil, fmt.Errorf("invalid unit name: %v", path)
}
obj := c.sysconn.Object("org.freedesktop.systemd1", path)
err = obj.CallWithContext(ctx, "org.freedesktop.DBus.Properties.GetAll", 0, dbusInterface).Store(&props)
if err != nil {
return nil, err
}
out := make(map[string]interface{}, len(props))
for k, v := range props {
out[k] = v.Value()
}
return out, nil
}
// Deprecated: use GetUnitPropertiesContext instead.
func (c *Conn) GetUnitProperties(unit string) (map[string]interface{}, error) {
return c.GetUnitPropertiesContext(context.Background(), unit)
}
// GetUnitPropertiesContext takes the (unescaped) unit name and returns all of
// its dbus object properties.
func (c *Conn) GetUnitPropertiesContext(ctx context.Context, unit string) (map[string]interface{}, error) {
path := unitPath(unit)
return c.getProperties(ctx, path, "org.freedesktop.systemd1.Unit")
}
// Deprecated: use GetUnitPathPropertiesContext instead.
func (c *Conn) GetUnitPathProperties(path dbus.ObjectPath) (map[string]interface{}, error) {
return c.GetUnitPathPropertiesContext(context.Background(), path)
}
// GetUnitPathPropertiesContext takes the (escaped) unit path and returns all
// of its dbus object properties.
func (c *Conn) GetUnitPathPropertiesContext(ctx context.Context, path dbus.ObjectPath) (map[string]interface{}, error) {
return c.getProperties(ctx, path, "org.freedesktop.systemd1.Unit")
}
// Deprecated: use GetAllPropertiesContext instead.
func (c *Conn) GetAllProperties(unit string) (map[string]interface{}, error) {
return c.GetAllPropertiesContext(context.Background(), unit)
}
// GetAllPropertiesContext takes the (unescaped) unit name and returns all of
// its dbus object properties.
func (c *Conn) GetAllPropertiesContext(ctx context.Context, unit string) (map[string]interface{}, error) {
path := unitPath(unit)
return c.getProperties(ctx, path, "")
}
func (c *Conn) getProperty(ctx context.Context, unit string, dbusInterface string, propertyName string) (*Property, error) {
var err error
var prop dbus.Variant
path := unitPath(unit)
if !path.IsValid() {
return nil, errors.New("invalid unit name: " + unit)
}
obj := c.sysconn.Object("org.freedesktop.systemd1", path)
err = obj.CallWithContext(ctx, "org.freedesktop.DBus.Properties.Get", 0, dbusInterface, propertyName).Store(&prop)
if err != nil {
return nil, err
}
return &Property{Name: propertyName, Value: prop}, nil
}
// Deprecated: use GetUnitPropertyContext instead.
func (c *Conn) GetUnitProperty(unit string, propertyName string) (*Property, error) {
return c.GetUnitPropertyContext(context.Background(), unit, propertyName)
}
// GetUnitPropertyContext takes an (unescaped) unit name, and a property name,
// and returns the property value.
func (c *Conn) GetUnitPropertyContext(ctx context.Context, unit string, propertyName string) (*Property, error) {
return c.getProperty(ctx, unit, "org.freedesktop.systemd1.Unit", propertyName)
}
// Deprecated: use GetServicePropertyContext instead.
func (c *Conn) GetServiceProperty(service string, propertyName string) (*Property, error) {
return c.GetServicePropertyContext(context.Background(), service, propertyName)
}
// GetServiceProperty returns property for given service name and property name.
func (c *Conn) GetServicePropertyContext(ctx context.Context, service string, propertyName string) (*Property, error) {
return c.getProperty(ctx, service, "org.freedesktop.systemd1.Service", propertyName)
}
// Deprecated: use GetUnitTypePropertiesContext instead.
func (c *Conn) GetUnitTypeProperties(unit string, unitType string) (map[string]interface{}, error) {
return c.GetUnitTypePropertiesContext(context.Background(), unit, unitType)
}
// GetUnitTypePropertiesContext returns the extra properties for a unit, specific to the unit type.
// Valid values for unitType: Service, Socket, Target, Device, Mount, Automount, Snapshot, Timer, Swap, Path, Slice, Scope.
// Returns "dbus.Error: Unknown interface" error if the unitType is not the correct type of the unit.
func (c *Conn) GetUnitTypePropertiesContext(ctx context.Context, unit string, unitType string) (map[string]interface{}, error) {
path := unitPath(unit)
return c.getProperties(ctx, path, "org.freedesktop.systemd1."+unitType)
}
// Deprecated: use SetUnitPropertiesContext instead.
func (c *Conn) SetUnitProperties(name string, runtime bool, properties ...Property) error {
return c.SetUnitPropertiesContext(context.Background(), name, runtime, properties...)
}
// SetUnitPropertiesContext may be used to modify certain unit properties at runtime.
// Not all properties may be changed at runtime, but many resource management
// settings (primarily those in systemd.cgroup(5)) may. The changes are applied
// instantly, and stored on disk for future boots, unless runtime is true, in which
// case the settings only apply until the next reboot. name is the name of the unit
// to modify. properties are the settings to set, encoded as an array of property
// name and value pairs.
func (c *Conn) SetUnitPropertiesContext(ctx context.Context, name string, runtime bool, properties ...Property) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.SetUnitProperties", 0, name, runtime, properties).Store()
}
// Deprecated: use GetUnitTypePropertyContext instead.
func (c *Conn) GetUnitTypeProperty(unit string, unitType string, propertyName string) (*Property, error) {
return c.GetUnitTypePropertyContext(context.Background(), unit, unitType, propertyName)
}
// GetUnitTypePropertyContext takes a property name, a unit name, and a unit type,
// and returns a property value. For valid values of unitType, see GetUnitTypePropertiesContext.
func (c *Conn) GetUnitTypePropertyContext(ctx context.Context, unit string, unitType string, propertyName string) (*Property, error) {
return c.getProperty(ctx, unit, "org.freedesktop.systemd1."+unitType, propertyName)
}
type UnitStatus struct {
Name string // The primary unit name as string
Description string // The human readable description string
LoadState string // The load state (i.e. whether the unit file has been loaded successfully)
ActiveState string // The active state (i.e. whether the unit is currently started or not)
SubState string // The sub state (a more fine-grained version of the active state that is specific to the unit type, which the active state is not)
Followed string // A unit that is being followed in its state by this unit, if there is any, otherwise the empty string.
Path dbus.ObjectPath // The unit object path
JobId uint32 // If there is a job queued for the job unit the numeric job id, 0 otherwise
JobType string // The job type as string
JobPath dbus.ObjectPath // The job object path
}
type storeFunc func(retvalues ...interface{}) error
func (c *Conn) listUnitsInternal(f storeFunc) ([]UnitStatus, error) {
result := make([][]interface{}, 0)
err := f(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
status := make([]UnitStatus, len(result))
statusInterface := make([]interface{}, len(status))
for i := range status {
statusInterface[i] = &status[i]
}
err = dbus.Store(resultInterface, statusInterface...)
if err != nil {
return nil, err
}
return status, nil
}
// GetUnitByPID returns the unit object path of the unit a process ID
// belongs to. It takes a UNIX PID and returns the object path. The PID must
// refer to an existing system process
func (c *Conn) GetUnitByPID(ctx context.Context, pid uint32) (dbus.ObjectPath, error) {
var result dbus.ObjectPath
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.GetUnitByPID", 0, pid).Store(&result)
return result, err
}
// GetUnitNameByPID returns the name of the unit a process ID belongs to. It
// takes a UNIX PID and returns the object path. The PID must refer to an
// existing system process
func (c *Conn) GetUnitNameByPID(ctx context.Context, pid uint32) (string, error) {
path, err := c.GetUnitByPID(ctx, pid)
if err != nil {
return "", err
}
return unitName(path), nil
}
// Deprecated: use ListUnitsContext instead.
func (c *Conn) ListUnits() ([]UnitStatus, error) {
return c.ListUnitsContext(context.Background())
}
// ListUnitsContext returns an array with all currently loaded units. Note that
// units may be known by multiple names at the same time, and hence there might
// be more unit names loaded than actual units behind them.
// Also note that a unit is only loaded if it is active and/or enabled.
// Units that are both disabled and inactive will thus not be returned.
func (c *Conn) ListUnitsContext(ctx context.Context) ([]UnitStatus, error) {
return c.listUnitsInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnits", 0).Store)
}
// Deprecated: use ListUnitsFilteredContext instead.
func (c *Conn) ListUnitsFiltered(states []string) ([]UnitStatus, error) {
return c.ListUnitsFilteredContext(context.Background(), states)
}
// ListUnitsFilteredContext returns an array with units filtered by state.
// It takes a list of units' statuses to filter.
func (c *Conn) ListUnitsFilteredContext(ctx context.Context, states []string) ([]UnitStatus, error) {
return c.listUnitsInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnitsFiltered", 0, states).Store)
}
// Deprecated: use ListUnitsByPatternsContext instead.
func (c *Conn) ListUnitsByPatterns(states []string, patterns []string) ([]UnitStatus, error) {
return c.ListUnitsByPatternsContext(context.Background(), states, patterns)
}
// ListUnitsByPatternsContext returns an array with units.
// It takes a list of units' statuses and names to filter.
// Note that units may be known by multiple names at the same time,
// and hence there might be more unit names loaded than actual units behind them.
func (c *Conn) ListUnitsByPatternsContext(ctx context.Context, states []string, patterns []string) ([]UnitStatus, error) {
return c.listUnitsInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnitsByPatterns", 0, states, patterns).Store)
}
// Deprecated: use ListUnitsByNamesContext instead.
func (c *Conn) ListUnitsByNames(units []string) ([]UnitStatus, error) {
return c.ListUnitsByNamesContext(context.Background(), units)
}
// ListUnitsByNamesContext returns an array with units. It takes a list of units'
// names and returns an UnitStatus array. Comparing to ListUnitsByPatternsContext
// method, this method returns statuses even for inactive or non-existing
// units. Input array should contain exact unit names, but not patterns.
//
// Requires systemd v230 or higher.
func (c *Conn) ListUnitsByNamesContext(ctx context.Context, units []string) ([]UnitStatus, error) {
return c.listUnitsInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnitsByNames", 0, units).Store)
}
type UnitFile struct {
Path string
Type string
}
func (c *Conn) listUnitFilesInternal(f storeFunc) ([]UnitFile, error) {
result := make([][]interface{}, 0)
err := f(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
files := make([]UnitFile, len(result))
fileInterface := make([]interface{}, len(files))
for i := range files {
fileInterface[i] = &files[i]
}
err = dbus.Store(resultInterface, fileInterface...)
if err != nil {
return nil, err
}
return files, nil
}
// Deprecated: use ListUnitFilesContext instead.
func (c *Conn) ListUnitFiles() ([]UnitFile, error) {
return c.ListUnitFilesContext(context.Background())
}
// ListUnitFiles returns an array of all available units on disk.
func (c *Conn) ListUnitFilesContext(ctx context.Context) ([]UnitFile, error) {
return c.listUnitFilesInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnitFiles", 0).Store)
}
// Deprecated: use ListUnitFilesByPatternsContext instead.
func (c *Conn) ListUnitFilesByPatterns(states []string, patterns []string) ([]UnitFile, error) {
return c.ListUnitFilesByPatternsContext(context.Background(), states, patterns)
}
// ListUnitFilesByPatternsContext returns an array of all available units on disk matched the patterns.
func (c *Conn) ListUnitFilesByPatternsContext(ctx context.Context, states []string, patterns []string) ([]UnitFile, error) {
return c.listUnitFilesInternal(c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListUnitFilesByPatterns", 0, states, patterns).Store)
}
type LinkUnitFileChange EnableUnitFileChange
// Deprecated: use LinkUnitFilesContext instead.
func (c *Conn) LinkUnitFiles(files []string, runtime bool, force bool) ([]LinkUnitFileChange, error) {
return c.LinkUnitFilesContext(context.Background(), files, runtime, force)
}
// LinkUnitFilesContext links unit files (that are located outside of the
// usual unit search paths) into the unit search path.
//
// It takes a list of absolute paths to unit files to link and two
// booleans.
//
// The first boolean controls whether the unit shall be
// enabled for runtime only (true, /run), or persistently (false,
// /etc).
//
// The second controls whether symlinks pointing to other units shall
// be replaced if necessary.
//
// This call returns a list of the changes made. The list consists of
// structures with three strings: the type of the change (one of symlink
// or unlink), the file name of the symlink and the destination of the
// symlink.
func (c *Conn) LinkUnitFilesContext(ctx context.Context, files []string, runtime bool, force bool) ([]LinkUnitFileChange, error) {
result := make([][]interface{}, 0)
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.LinkUnitFiles", 0, files, runtime, force).Store(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
changes := make([]LinkUnitFileChange, len(result))
changesInterface := make([]interface{}, len(changes))
for i := range changes {
changesInterface[i] = &changes[i]
}
err = dbus.Store(resultInterface, changesInterface...)
if err != nil {
return nil, err
}
return changes, nil
}
// Deprecated: use EnableUnitFilesContext instead.
func (c *Conn) EnableUnitFiles(files []string, runtime bool, force bool) (bool, []EnableUnitFileChange, error) {
return c.EnableUnitFilesContext(context.Background(), files, runtime, force)
}
// EnableUnitFilesContext may be used to enable one or more units in the system
// (by creating symlinks to them in /etc or /run).
//
// It takes a list of unit files to enable (either just file names or full
// absolute paths if the unit files are residing outside the usual unit
// search paths), and two booleans: the first controls whether the unit shall
// be enabled for runtime only (true, /run), or persistently (false, /etc).
// The second one controls whether symlinks pointing to other units shall
// be replaced if necessary.
//
// This call returns one boolean and an array with the changes made. The
// boolean signals whether the unit files contained any enablement
// information (i.e. an [Install]) section. The changes list consists of
// structures with three strings: the type of the change (one of symlink
// or unlink), the file name of the symlink and the destination of the
// symlink.
func (c *Conn) EnableUnitFilesContext(ctx context.Context, files []string, runtime bool, force bool) (bool, []EnableUnitFileChange, error) {
var carries_install_info bool
result := make([][]interface{}, 0)
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.EnableUnitFiles", 0, files, runtime, force).Store(&carries_install_info, &result)
if err != nil {
return false, nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
changes := make([]EnableUnitFileChange, len(result))
changesInterface := make([]interface{}, len(changes))
for i := range changes {
changesInterface[i] = &changes[i]
}
err = dbus.Store(resultInterface, changesInterface...)
if err != nil {
return false, nil, err
}
return carries_install_info, changes, nil
}
type EnableUnitFileChange struct {
Type string // Type of the change (one of symlink or unlink)
Filename string // File name of the symlink
Destination string // Destination of the symlink
}
// Deprecated: use DisableUnitFilesContext instead.
func (c *Conn) DisableUnitFiles(files []string, runtime bool) ([]DisableUnitFileChange, error) {
return c.DisableUnitFilesContext(context.Background(), files, runtime)
}
// DisableUnitFilesContext may be used to disable one or more units in the
// system (by removing symlinks to them from /etc or /run).
//
// It takes a list of unit files to disable (either just file names or full
// absolute paths if the unit files are residing outside the usual unit
// search paths), and one boolean: whether the unit was enabled for runtime
// only (true, /run), or persistently (false, /etc).
//
// This call returns an array with the changes made. The changes list
// consists of structures with three strings: the type of the change (one of
// symlink or unlink), the file name of the symlink and the destination of the
// symlink.
func (c *Conn) DisableUnitFilesContext(ctx context.Context, files []string, runtime bool) ([]DisableUnitFileChange, error) {
result := make([][]interface{}, 0)
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.DisableUnitFiles", 0, files, runtime).Store(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
changes := make([]DisableUnitFileChange, len(result))
changesInterface := make([]interface{}, len(changes))
for i := range changes {
changesInterface[i] = &changes[i]
}
err = dbus.Store(resultInterface, changesInterface...)
if err != nil {
return nil, err
}
return changes, nil
}
type DisableUnitFileChange struct {
Type string // Type of the change (one of symlink or unlink)
Filename string // File name of the symlink
Destination string // Destination of the symlink
}
// Deprecated: use MaskUnitFilesContext instead.
func (c *Conn) MaskUnitFiles(files []string, runtime bool, force bool) ([]MaskUnitFileChange, error) {
return c.MaskUnitFilesContext(context.Background(), files, runtime, force)
}
// MaskUnitFilesContext masks one or more units in the system.
//
// The files argument contains a list of units to mask (either just file names
// or full absolute paths if the unit files are residing outside the usual unit
// search paths).
//
// The runtime argument is used to specify whether the unit was enabled for
// runtime only (true, /run/systemd/..), or persistently (false,
// /etc/systemd/..).
func (c *Conn) MaskUnitFilesContext(ctx context.Context, files []string, runtime bool, force bool) ([]MaskUnitFileChange, error) {
result := make([][]interface{}, 0)
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.MaskUnitFiles", 0, files, runtime, force).Store(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
changes := make([]MaskUnitFileChange, len(result))
changesInterface := make([]interface{}, len(changes))
for i := range changes {
changesInterface[i] = &changes[i]
}
err = dbus.Store(resultInterface, changesInterface...)
if err != nil {
return nil, err
}
return changes, nil
}
type MaskUnitFileChange struct {
Type string // Type of the change (one of symlink or unlink)
Filename string // File name of the symlink
Destination string // Destination of the symlink
}
// Deprecated: use UnmaskUnitFilesContext instead.
func (c *Conn) UnmaskUnitFiles(files []string, runtime bool) ([]UnmaskUnitFileChange, error) {
return c.UnmaskUnitFilesContext(context.Background(), files, runtime)
}
// UnmaskUnitFilesContext unmasks one or more units in the system.
//
// It takes the list of unit files to mask (either just file names or full
// absolute paths if the unit files are residing outside the usual unit search
// paths), and a boolean runtime flag to specify whether the unit was enabled
// for runtime only (true, /run/systemd/..), or persistently (false,
// /etc/systemd/..).
func (c *Conn) UnmaskUnitFilesContext(ctx context.Context, files []string, runtime bool) ([]UnmaskUnitFileChange, error) {
result := make([][]interface{}, 0)
err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.UnmaskUnitFiles", 0, files, runtime).Store(&result)
if err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
changes := make([]UnmaskUnitFileChange, len(result))
changesInterface := make([]interface{}, len(changes))
for i := range changes {
changesInterface[i] = &changes[i]
}
err = dbus.Store(resultInterface, changesInterface...)
if err != nil {
return nil, err
}
return changes, nil
}
type UnmaskUnitFileChange struct {
Type string // Type of the change (one of symlink or unlink)
Filename string // File name of the symlink
Destination string // Destination of the symlink
}
// Deprecated: use ReloadContext instead.
func (c *Conn) Reload() error {
return c.ReloadContext(context.Background())
}
// ReloadContext instructs systemd to scan for and reload unit files. This is
// an equivalent to systemctl daemon-reload.
func (c *Conn) ReloadContext(ctx context.Context) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.Reload", 0).Store()
}
func unitPath(name string) dbus.ObjectPath {
return dbus.ObjectPath("/org/freedesktop/systemd1/unit/" + PathBusEscape(name))
}
// unitName returns the unescaped base element of the supplied escaped path.
func unitName(dpath dbus.ObjectPath) string {
return pathBusUnescape(path.Base(string(dpath)))
}
// JobStatus holds a currently queued job definition.
type JobStatus struct {
Id uint32 // The numeric job id
Unit string // The primary unit name for this job
JobType string // The job type as string
Status string // The job state as string
JobPath dbus.ObjectPath // The job object path
UnitPath dbus.ObjectPath // The unit object path
}
// Deprecated: use ListJobsContext instead.
func (c *Conn) ListJobs() ([]JobStatus, error) {
return c.ListJobsContext(context.Background())
}
// ListJobsContext returns an array with all currently queued jobs.
func (c *Conn) ListJobsContext(ctx context.Context) ([]JobStatus, error) {
return c.listJobsInternal(ctx)
}
func (c *Conn) listJobsInternal(ctx context.Context) ([]JobStatus, error) {
result := make([][]interface{}, 0)
if err := c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ListJobs", 0).Store(&result); err != nil {
return nil, err
}
resultInterface := make([]interface{}, len(result))
for i := range result {
resultInterface[i] = result[i]
}
status := make([]JobStatus, len(result))
statusInterface := make([]interface{}, len(status))
for i := range status {
statusInterface[i] = &status[i]
}
if err := dbus.Store(resultInterface, statusInterface...); err != nil {
return nil, err
}
return status, nil
}
// Freeze the cgroup associated with the unit.
// Note that FreezeUnit and ThawUnit are only supported on systems running with cgroup v2.
func (c *Conn) FreezeUnit(ctx context.Context, unit string) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.FreezeUnit", 0, unit).Store()
}
// Unfreeze the cgroup associated with the unit.
func (c *Conn) ThawUnit(ctx context.Context, unit string) error {
return c.sysobj.CallWithContext(ctx, "org.freedesktop.systemd1.Manager.ThawUnit", 0, unit).Store()
}
-237
View File
@@ -1,237 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbus
import (
"github.com/godbus/dbus/v5"
)
// From the systemd docs:
//
// The properties array of StartTransientUnit() may take many of the settings
// that may also be configured in unit files. Not all parameters are currently
// accepted though, but we plan to cover more properties with future release.
// Currently you may set the Description, Slice and all dependency types of
// units, as well as RemainAfterExit, ExecStart for service units,
// TimeoutStopUSec and PIDs for scope units, and CPUAccounting, CPUShares,
// BlockIOAccounting, BlockIOWeight, BlockIOReadBandwidth,
// BlockIOWriteBandwidth, BlockIODeviceWeight, MemoryAccounting, MemoryLimit,
// DevicePolicy, DeviceAllow for services/scopes/slices. These fields map
// directly to their counterparts in unit files and as normal D-Bus object
// properties. The exception here is the PIDs field of scope units which is
// used for construction of the scope only and specifies the initial PIDs to
// add to the scope object.
type Property struct {
Name string
Value dbus.Variant
}
type PropertyCollection struct {
Name string
Properties []Property
}
type execStart struct {
Path string // the binary path to execute
Args []string // an array with all arguments to pass to the executed command, starting with argument 0
UncleanIsFailure bool // a boolean whether it should be considered a failure if the process exits uncleanly
}
// PropExecStart sets the ExecStart service property. The first argument is a
// slice with the binary path to execute followed by the arguments to pass to
// the executed command. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#ExecStart=
func PropExecStart(command []string, uncleanIsFailure bool) Property {
execStarts := []execStart{
{
Path: command[0],
Args: command,
UncleanIsFailure: uncleanIsFailure,
},
}
return Property{
Name: "ExecStart",
Value: dbus.MakeVariant(execStarts),
}
}
// PropRemainAfterExit sets the RemainAfterExit service property. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#RemainAfterExit=
func PropRemainAfterExit(b bool) Property {
return Property{
Name: "RemainAfterExit",
Value: dbus.MakeVariant(b),
}
}
// PropType sets the Type service property. See
// http://www.freedesktop.org/software/systemd/man/systemd.service.html#Type=
func PropType(t string) Property {
return Property{
Name: "Type",
Value: dbus.MakeVariant(t),
}
}
// PropDescription sets the Description unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit#Description=
func PropDescription(desc string) Property {
return Property{
Name: "Description",
Value: dbus.MakeVariant(desc),
}
}
func propDependency(name string, units []string) Property {
return Property{
Name: name,
Value: dbus.MakeVariant(units),
}
}
// PropRequires sets the Requires unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requires=
func PropRequires(units ...string) Property {
return propDependency("Requires", units)
}
// PropRequiresOverridable sets the RequiresOverridable unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresOverridable=
func PropRequiresOverridable(units ...string) Property {
return propDependency("RequiresOverridable", units)
}
// PropRequisite sets the Requisite unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Requisite=
func PropRequisite(units ...string) Property {
return propDependency("Requisite", units)
}
// PropRequisiteOverridable sets the RequisiteOverridable unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequisiteOverridable=
func PropRequisiteOverridable(units ...string) Property {
return propDependency("RequisiteOverridable", units)
}
// PropWants sets the Wants unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Wants=
func PropWants(units ...string) Property {
return propDependency("Wants", units)
}
// PropBindsTo sets the BindsTo unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#BindsTo=
func PropBindsTo(units ...string) Property {
return propDependency("BindsTo", units)
}
// PropRequiredBy sets the RequiredBy unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredBy=
func PropRequiredBy(units ...string) Property {
return propDependency("RequiredBy", units)
}
// PropRequiredByOverridable sets the RequiredByOverridable unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiredByOverridable=
func PropRequiredByOverridable(units ...string) Property {
return propDependency("RequiredByOverridable", units)
}
// PropWantedBy sets the WantedBy unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#WantedBy=
func PropWantedBy(units ...string) Property {
return propDependency("WantedBy", units)
}
// PropBoundBy sets the BoundBy unit property. See
// http://www.freedesktop.org/software/systemd/main/systemd.unit.html#BoundBy=
func PropBoundBy(units ...string) Property {
return propDependency("BoundBy", units)
}
// PropConflicts sets the Conflicts unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Conflicts=
func PropConflicts(units ...string) Property {
return propDependency("Conflicts", units)
}
// PropConflictedBy sets the ConflictedBy unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#ConflictedBy=
func PropConflictedBy(units ...string) Property {
return propDependency("ConflictedBy", units)
}
// PropBefore sets the Before unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Before=
func PropBefore(units ...string) Property {
return propDependency("Before", units)
}
// PropAfter sets the After unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#After=
func PropAfter(units ...string) Property {
return propDependency("After", units)
}
// PropOnFailure sets the OnFailure unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#OnFailure=
func PropOnFailure(units ...string) Property {
return propDependency("OnFailure", units)
}
// PropTriggers sets the Triggers unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#Triggers=
func PropTriggers(units ...string) Property {
return propDependency("Triggers", units)
}
// PropTriggeredBy sets the TriggeredBy unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#TriggeredBy=
func PropTriggeredBy(units ...string) Property {
return propDependency("TriggeredBy", units)
}
// PropPropagatesReloadTo sets the PropagatesReloadTo unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#PropagatesReloadTo=
func PropPropagatesReloadTo(units ...string) Property {
return propDependency("PropagatesReloadTo", units)
}
// PropRequiresMountsFor sets the RequiresMountsFor unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.unit.html#RequiresMountsFor=
func PropRequiresMountsFor(units ...string) Property {
return propDependency("RequiresMountsFor", units)
}
// PropSlice sets the Slice unit property. See
// http://www.freedesktop.org/software/systemd/man/systemd.resource-control.html#Slice=
func PropSlice(slice string) Property {
return Property{
Name: "Slice",
Value: dbus.MakeVariant(slice),
}
}
// PropPids sets the PIDs field of scope units used in the initial construction
// of the scope only and specifies the initial PIDs to add to the scope object.
// See https://www.freedesktop.org/wiki/Software/systemd/ControlGroupInterface/#properties
func PropPids(pids ...uint32) Property {
return Property{
Name: "PIDs",
Value: dbus.MakeVariant(pids),
}
}
-47
View File
@@ -1,47 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbus
type set struct {
data map[string]bool
}
func (s *set) Add(value string) {
s.data[value] = true
}
func (s *set) Remove(value string) {
delete(s.data, value)
}
func (s *set) Contains(value string) (exists bool) {
_, exists = s.data[value]
return
}
func (s *set) Length() int {
return len(s.data)
}
func (s *set) Values() (values []string) {
for val := range s.data {
values = append(values, val)
}
return
}
func newSet() *set {
return &set{make(map[string]bool)}
}
-333
View File
@@ -1,333 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbus
import (
"errors"
"log"
"time"
"github.com/godbus/dbus/v5"
)
const (
cleanIgnoreInterval = int64(10 * time.Second)
ignoreInterval = int64(30 * time.Millisecond)
)
// Subscribe sets up this connection to subscribe to all systemd dbus events.
// This is required before calling SubscribeUnits. When the connection closes
// systemd will automatically stop sending signals so there is no need to
// explicitly call Unsubscribe().
func (c *Conn) Subscribe() error {
c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
"type='signal',interface='org.freedesktop.systemd1.Manager',member='UnitNew'")
c.sigconn.BusObject().Call("org.freedesktop.DBus.AddMatch", 0,
"type='signal',interface='org.freedesktop.DBus.Properties',member='PropertiesChanged'")
return c.sigobj.Call("org.freedesktop.systemd1.Manager.Subscribe", 0).Store()
}
// Unsubscribe this connection from systemd dbus events.
func (c *Conn) Unsubscribe() error {
return c.sigobj.Call("org.freedesktop.systemd1.Manager.Unsubscribe", 0).Store()
}
func (c *Conn) dispatch() {
ch := make(chan *dbus.Signal, signalBuffer)
c.sigconn.Signal(ch)
go func() {
for {
signal, ok := <-ch
if !ok {
return
}
if signal.Name == "org.freedesktop.systemd1.Manager.JobRemoved" {
c.jobComplete(signal)
}
if c.subStateSubscriber.updateCh == nil &&
c.propertiesSubscriber.updateCh == nil {
continue
}
var unitPath dbus.ObjectPath
switch signal.Name {
case "org.freedesktop.systemd1.Manager.JobRemoved":
unitName := signal.Body[2].(string)
c.sysobj.Call("org.freedesktop.systemd1.Manager.GetUnit", 0, unitName).Store(&unitPath)
case "org.freedesktop.systemd1.Manager.UnitNew":
unitPath = signal.Body[1].(dbus.ObjectPath)
case "org.freedesktop.DBus.Properties.PropertiesChanged":
if signal.Body[0].(string) == "org.freedesktop.systemd1.Unit" {
unitPath = signal.Path
if len(signal.Body) >= 2 {
if changed, ok := signal.Body[1].(map[string]dbus.Variant); ok {
c.sendPropertiesUpdate(unitPath, changed)
}
}
}
}
if unitPath == dbus.ObjectPath("") {
continue
}
c.sendSubStateUpdate(unitPath)
}
}()
}
// SubscribeUnits returns two unbuffered channels which will receive all changed units every
// interval. Deleted units are sent as nil.
func (c *Conn) SubscribeUnits(interval time.Duration) (<-chan map[string]*UnitStatus, <-chan error) {
return c.SubscribeUnitsCustom(interval, 0, func(u1, u2 *UnitStatus) bool { return *u1 != *u2 }, nil)
}
// SubscribeUnitsCustom is like SubscribeUnits but lets you specify the buffer
// size of the channels, the comparison function for detecting changes and a filter
// function for cutting down on the noise that your channel receives.
func (c *Conn) SubscribeUnitsCustom(interval time.Duration, buffer int, isChanged func(*UnitStatus, *UnitStatus) bool, filterUnit func(string) bool) (<-chan map[string]*UnitStatus, <-chan error) {
old := make(map[string]*UnitStatus)
statusChan := make(chan map[string]*UnitStatus, buffer)
errChan := make(chan error, buffer)
go func() {
for {
timerChan := time.After(interval)
units, err := c.ListUnits()
if err == nil {
cur := make(map[string]*UnitStatus)
for i := range units {
if filterUnit != nil && filterUnit(units[i].Name) {
continue
}
cur[units[i].Name] = &units[i]
}
// add all new or changed units
changed := make(map[string]*UnitStatus)
for n, u := range cur {
if oldU, ok := old[n]; !ok || isChanged(oldU, u) {
changed[n] = u
}
delete(old, n)
}
// add all deleted units
for oldN := range old {
changed[oldN] = nil
}
old = cur
if len(changed) != 0 {
statusChan <- changed
}
} else {
errChan <- err
}
<-timerChan
}
}()
return statusChan, errChan
}
type SubStateUpdate struct {
UnitName string
SubState string
}
// SetSubStateSubscriber writes to updateCh when any unit's substate changes.
// Although this writes to updateCh on every state change, the reported state
// may be more recent than the change that generated it (due to an unavoidable
// race in the systemd dbus interface). That is, this method provides a good
// way to keep a current view of all units' states, but is not guaranteed to
// show every state transition they go through. Furthermore, state changes
// will only be written to the channel with non-blocking writes. If updateCh
// is full, it attempts to write an error to errCh; if errCh is full, the error
// passes silently.
func (c *Conn) SetSubStateSubscriber(updateCh chan<- *SubStateUpdate, errCh chan<- error) {
if c == nil {
msg := "nil receiver"
select {
case errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
c.subStateSubscriber.Lock()
defer c.subStateSubscriber.Unlock()
c.subStateSubscriber.updateCh = updateCh
c.subStateSubscriber.errCh = errCh
}
func (c *Conn) sendSubStateUpdate(unitPath dbus.ObjectPath) {
c.subStateSubscriber.Lock()
defer c.subStateSubscriber.Unlock()
if c.subStateSubscriber.updateCh == nil {
return
}
isIgnored := c.shouldIgnore(unitPath)
defer c.cleanIgnore()
if isIgnored {
return
}
info, err := c.GetUnitPathProperties(unitPath)
if err != nil {
select {
case c.subStateSubscriber.errCh <- err:
default:
log.Printf("full error channel while reporting: %s\n", err)
}
return
}
defer c.updateIgnore(unitPath, info)
name, ok := info["Id"].(string)
if !ok {
msg := "failed to cast info.Id"
select {
case c.subStateSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", err)
}
return
}
substate, ok := info["SubState"].(string)
if !ok {
msg := "failed to cast info.SubState"
select {
case c.subStateSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
update := &SubStateUpdate{name, substate}
select {
case c.subStateSubscriber.updateCh <- update:
default:
msg := "update channel is full"
select {
case c.subStateSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
}
// The ignore functions work around a wart in the systemd dbus interface.
// Requesting the properties of an unloaded unit will cause systemd to send a
// pair of UnitNew/UnitRemoved signals. Because we need to get a unit's
// properties on UnitNew (as that's the only indication of a new unit coming up
// for the first time), we would enter an infinite loop if we did not attempt
// to detect and ignore these spurious signals. The signal themselves are
// indistinguishable from relevant ones, so we (somewhat hackishly) ignore an
// unloaded unit's signals for a short time after requesting its properties.
// This means that we will miss e.g. a transient unit being restarted
// *immediately* upon failure and also a transient unit being started
// immediately after requesting its status (with systemctl status, for example,
// because this causes a UnitNew signal to be sent which then causes us to fetch
// the properties).
func (c *Conn) shouldIgnore(path dbus.ObjectPath) bool {
t, ok := c.subStateSubscriber.ignore[path]
return ok && t >= time.Now().UnixNano()
}
func (c *Conn) updateIgnore(path dbus.ObjectPath, info map[string]interface{}) {
loadState, ok := info["LoadState"].(string)
if !ok {
return
}
// unit is unloaded - it will trigger bad systemd dbus behavior
if loadState == "not-found" {
c.subStateSubscriber.ignore[path] = time.Now().UnixNano() + ignoreInterval
}
}
// without this, ignore would grow unboundedly over time
func (c *Conn) cleanIgnore() {
now := time.Now().UnixNano()
if c.subStateSubscriber.cleanIgnore < now {
c.subStateSubscriber.cleanIgnore = now + cleanIgnoreInterval
for p, t := range c.subStateSubscriber.ignore {
if t < now {
delete(c.subStateSubscriber.ignore, p)
}
}
}
}
// PropertiesUpdate holds a map of a unit's changed properties
type PropertiesUpdate struct {
UnitName string
Changed map[string]dbus.Variant
}
// SetPropertiesSubscriber writes to updateCh when any unit's properties
// change. Every property change reported by systemd will be sent; that is, no
// transitions will be "missed" (as they might be with SetSubStateSubscriber).
// However, state changes will only be written to the channel with non-blocking
// writes. If updateCh is full, it attempts to write an error to errCh; if
// errCh is full, the error passes silently.
func (c *Conn) SetPropertiesSubscriber(updateCh chan<- *PropertiesUpdate, errCh chan<- error) {
c.propertiesSubscriber.Lock()
defer c.propertiesSubscriber.Unlock()
c.propertiesSubscriber.updateCh = updateCh
c.propertiesSubscriber.errCh = errCh
}
// we don't need to worry about shouldIgnore() here because
// sendPropertiesUpdate doesn't call GetProperties()
func (c *Conn) sendPropertiesUpdate(unitPath dbus.ObjectPath, changedProps map[string]dbus.Variant) {
c.propertiesSubscriber.Lock()
defer c.propertiesSubscriber.Unlock()
if c.propertiesSubscriber.updateCh == nil {
return
}
update := &PropertiesUpdate{unitName(unitPath), changedProps}
select {
case c.propertiesSubscriber.updateCh <- update:
default:
msg := "update channel is full"
select {
case c.propertiesSubscriber.errCh <- errors.New(msg):
default:
log.Printf("full error channel while reporting: %s\n", msg)
}
return
}
}
-57
View File
@@ -1,57 +0,0 @@
// Copyright 2015 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package dbus
import (
"time"
)
// SubscriptionSet returns a subscription set which is like conn.Subscribe but
// can filter to only return events for a set of units.
type SubscriptionSet struct {
*set
conn *Conn
}
func (s *SubscriptionSet) filter(unit string) bool {
return !s.Contains(unit)
}
// Subscribe starts listening for dbus events for all of the units in the set.
// Returns channels identical to conn.SubscribeUnits.
func (s *SubscriptionSet) Subscribe() (<-chan map[string]*UnitStatus, <-chan error) {
// TODO: Make fully evented by using systemd 209 with properties changed values
return s.conn.SubscribeUnitsCustom(time.Second, 0,
mismatchUnitStatus,
func(unit string) bool { return s.filter(unit) },
)
}
// NewSubscriptionSet returns a new subscription set.
func (conn *Conn) NewSubscriptionSet() *SubscriptionSet {
return &SubscriptionSet{newSet(), conn}
}
// mismatchUnitStatus returns true if the provided UnitStatus objects
// are not equivalent. false is returned if the objects are equivalent.
// Only the Name, Description and state-related fields are used in
// the comparison.
func mismatchUnitStatus(u1, u2 *UnitStatus) bool {
return u1.Name != u2.Name ||
u1.Description != u2.Description ||
u1.LoadState != u2.LoadState ||
u1.ActiveState != u2.ActiveState ||
u1.SubState != u2.SubState
}
+9 -4
View File
@@ -269,7 +269,7 @@ func (c *Color) Printf(format string, a ...interface{}) (n int, err error) {
// On Windows, users should wrap w with colorable.NewColorable() if w is of
// type *os.File.
func (c *Color) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
return fmt.Fprintln(w, c.wrap(fmt.Sprint(a...)))
return fmt.Fprintln(w, c.wrap(sprintln(a...)))
}
// Println formats using the default formats for its operands and writes to
@@ -278,7 +278,7 @@ func (c *Color) Fprintln(w io.Writer, a ...interface{}) (n int, err error) {
// encountered. This is the standard fmt.Print() method wrapped with the given
// color.
func (c *Color) Println(a ...interface{}) (n int, err error) {
return fmt.Fprintln(Output, c.wrap(fmt.Sprint(a...)))
return fmt.Fprintln(Output, c.wrap(sprintln(a...)))
}
// Sprint is just like Print, but returns a string instead of printing it.
@@ -288,7 +288,7 @@ func (c *Color) Sprint(a ...interface{}) string {
// Sprintln is just like Println, but returns a string instead of printing it.
func (c *Color) Sprintln(a ...interface{}) string {
return fmt.Sprintln(c.Sprint(a...))
return c.wrap(sprintln(a...)) + "\n"
}
// Sprintf is just like Printf, but returns a string instead of printing it.
@@ -370,7 +370,7 @@ func (c *Color) SprintfFunc() func(format string, a ...interface{}) string {
// string. Windows users should use this in conjunction with color.Output.
func (c *Color) SprintlnFunc() func(a ...interface{}) string {
return func(a ...interface{}) string {
return fmt.Sprintln(c.Sprint(a...))
return c.wrap(sprintln(a...)) + "\n"
}
}
@@ -648,3 +648,8 @@ func HiCyanString(format string, a ...interface{}) string { return colorString(f
func HiWhiteString(format string, a ...interface{}) string {
return colorString(format, FgHiWhite, a...)
}
// sprintln is a helper function to format a string with fmt.Sprintln and trim the trailing newline.
func sprintln(a ...interface{}) string {
return strings.TrimSuffix(fmt.Sprintln(a...), "\n")
}
+19 -26
View File
@@ -13,7 +13,8 @@ import (
var (
sqliteSeparator = "`|\"|'|\t"
indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]? ON (.*)$`, sqliteSeparator, sqliteSeparator))
uniqueRegexp = regexp.MustCompile(fmt.Sprintf(`^CONSTRAINT [%v]?[\w-]+[%v]? UNIQUE (.*)$`, sqliteSeparator, sqliteSeparator))
indexRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)CREATE(?: UNIQUE)? INDEX [%v]?[\w\d-]+[%v]?(?s:.*?)ON (.*)$`, sqliteSeparator, sqliteSeparator))
tableRegexp = regexp.MustCompile(fmt.Sprintf(`(?is)(CREATE TABLE [%v]?[\w\d-]+[%v]?)(?:\s*\((.*)\))?`, sqliteSeparator, sqliteSeparator))
separatorRegexp = regexp.MustCompile(fmt.Sprintf("[%v]", sqliteSeparator))
columnsRegexp = regexp.MustCompile(fmt.Sprintf(`[(,][%v]?(\w+)[%v]?`, sqliteSeparator, sqliteSeparator))
@@ -103,11 +104,24 @@ func parseDDL(strs ...string) (*ddl, error) {
for _, f := range result.fields {
fUpper := strings.ToUpper(f)
if strings.HasPrefix(fUpper, "CHECK") ||
strings.HasPrefix(fUpper, "CONSTRAINT") {
if strings.HasPrefix(fUpper, "CHECK") {
continue
}
if strings.HasPrefix(fUpper, "CONSTRAINT") {
matches := uniqueRegexp.FindStringSubmatch(f)
if len(matches) > 0 {
if columns := getAllColumns(matches[1]); len(columns) == 1 {
for idx, column := range result.columns {
if column.NameValue.String == columns[0] {
column.UniqueValue = sql.NullBool{Bool: true, Valid: true}
result.columns[idx] = column
break
}
}
}
}
continue
}
if strings.HasPrefix(fUpper, "PRIMARY KEY") {
for _, name := range getAllColumns(f) {
for idx, column := range result.columns {
@@ -159,14 +173,7 @@ func parseDDL(strs ...string) (*ddl, error) {
}
}
} else if matches := indexRegexp.FindStringSubmatch(str); len(matches) > 0 {
for _, column := range getAllColumns(matches[1]) {
for idx, c := range result.columns {
if c.NameValue.String == column {
c.UniqueValue = sql.NullBool{Bool: strings.ToUpper(strings.Fields(str)[1]) == "UNIQUE", Valid: true}
result.columns[idx] = c
}
}
}
// don't report Unique by UniqueIndex
} else {
return nil, errors.New("invalid DDL")
}
@@ -268,20 +275,6 @@ func (d *ddl) getColumns() []string {
return res
}
func (d *ddl) alterColumn(name, sql string) bool {
reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$")
for i := 0; i < len(d.fields); i++ {
if reg.MatchString(d.fields[i]) {
d.fields[i] = sql
return false
}
}
d.fields = append(d.fields, sql)
return true
}
func (d *ddl) removeColumn(name string) bool {
reg := regexp.MustCompile("^(`|'|\"| )" + regexp.QuoteMeta(name) + "(`|'|\"| ) .*?$")
+64 -40
View File
@@ -79,14 +79,28 @@ func (m Migrator) AlterColumn(value interface{}, name string) error {
return m.RunWithoutForeignKey(func() error {
return m.recreateTable(value, nil, func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) {
if field := stmt.Schema.LookUpField(name); field != nil {
if ddl.alterColumn(field.DBName, fmt.Sprintf("`%s` ?", field.DBName)) {
return nil, nil, fmt.Errorf("field `%s` not found in origin ddl, ddl= '%s'", name, ddl.compile())
var sqlArgs []interface{}
for i, f := range ddl.fields {
if matches := columnRegexp.FindStringSubmatch(f); len(matches) > 1 && matches[1] == field.DBName {
ddl.fields[i] = fmt.Sprintf("`%v` ?", field.DBName)
sqlArgs = []interface{}{m.FullDataTypeOf(field)}
// table created by old version might look like `CREATE TABLE ? (? varchar(10) UNIQUE)`.
// FullDataTypeOf doesn't contain UNIQUE, so we need to add unique constraint.
if strings.Contains(strings.ToUpper(matches[3]), " UNIQUE") {
uniName := m.DB.NamingStrategy.UniqueName(stmt.Table, field.DBName)
uni, _ := m.GuessConstraintInterfaceAndTable(stmt, uniName)
if uni != nil {
uniSQL, uniArgs := uni.Build()
ddl.addConstraint(uniName, uniSQL)
sqlArgs = append(sqlArgs, uniArgs...)
}
}
break
}
}
return ddl, []interface{}{m.FullDataTypeOf(field)}, nil
return ddl, sqlArgs, nil
}
return nil, nil, fmt.Errorf("failed to alter field with name `%s`", name)
return nil, nil, fmt.Errorf("failed to alter field with name %v", name)
})
})
}
@@ -153,7 +167,7 @@ func (m Migrator) DropColumn(value interface{}, name string) error {
func (m Migrator) CreateConstraint(value interface{}, name string) error {
return m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)
return m.recreateTable(value, &table,
func(ddl *ddl, stmt *gorm.Statement) (*ddl, []interface{}, error) {
@@ -164,12 +178,8 @@ func (m Migrator) CreateConstraint(value interface{}, name string) error {
)
if constraint != nil {
constraintName = constraint.Name
constraintSql, constraintValues = buildConstraint(constraint)
} else if chk != nil {
constraintName = chk.Name
constraintSql = "CONSTRAINT ? CHECK (?)"
constraintValues = []interface{}{clause.Column{Name: chk.Name}, clause.Expr{SQL: chk.Constraint}}
constraintName = constraint.GetName()
constraintSql, constraintValues = constraint.Build()
} else {
return nil, nil, nil
}
@@ -182,11 +192,9 @@ func (m Migrator) CreateConstraint(value interface{}, name string) error {
func (m Migrator) DropConstraint(value interface{}, name string) error {
return m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)
if constraint != nil {
name = constraint.Name
} else if chk != nil {
name = chk.Name
name = constraint.GetName()
}
return m.recreateTable(value, &table,
@@ -200,11 +208,9 @@ func (m Migrator) DropConstraint(value interface{}, name string) error {
func (m Migrator) HasConstraint(value interface{}, name string) bool {
var count int64
m.RunWithValue(value, func(stmt *gorm.Statement) error {
constraint, chk, table := m.GuessConstraintAndTable(stmt, name)
constraint, table := m.GuessConstraintInterfaceAndTable(stmt, name)
if constraint != nil {
name = constraint.Name
} else if chk != nil {
name = chk.Name
name = constraint.GetName()
}
m.DB.Raw(
@@ -317,26 +323,44 @@ func (m Migrator) DropIndex(value interface{}, name string) error {
})
}
func buildConstraint(constraint *schema.Constraint) (sql string, results []interface{}) {
sql = "CONSTRAINT ? FOREIGN KEY ? REFERENCES ??"
if constraint.OnDelete != "" {
sql += " ON DELETE " + constraint.OnDelete
}
type Index struct {
Seq int
Name string
Unique bool
Origin string
Partial bool
}
if constraint.OnUpdate != "" {
sql += " ON UPDATE " + constraint.OnUpdate
}
var foreignKeys, references []interface{}
for _, field := range constraint.ForeignKeys {
foreignKeys = append(foreignKeys, clause.Column{Name: field.DBName})
}
for _, field := range constraint.References {
references = append(references, clause.Column{Name: field.DBName})
}
results = append(results, clause.Table{Name: constraint.Name}, foreignKeys, clause.Table{Name: constraint.ReferenceSchema.Table}, references)
return
// GetIndexes return Indexes []gorm.Index and execErr error,
// See the [doc]
//
// [doc]: https://www.sqlite.org/pragma.html#pragma_index_list
func (m Migrator) GetIndexes(value interface{}) ([]gorm.Index, error) {
indexes := make([]gorm.Index, 0)
err := m.RunWithValue(value, func(stmt *gorm.Statement) error {
rst := make([]*Index, 0)
if err := m.DB.Debug().Raw("SELECT * FROM PRAGMA_index_list(?)", stmt.Table).Scan(&rst).Error; err != nil { // alias `PRAGMA index_list(?)`
return err
}
for _, index := range rst {
if index.Origin == "u" { // skip the index was created by a UNIQUE constraint
continue
}
var columns []string
if err := m.DB.Raw("SELECT name FROM PRAGMA_index_info(?)", index.Name).Scan(&columns).Error; err != nil { // alias `PRAGMA index_info(?)`
return err
}
indexes = append(indexes, &migrator.Index{
TableName: stmt.Table,
NameValue: index.Name,
ColumnList: columns,
PrimaryKeyValue: sql.NullBool{Bool: index.Origin == "pk", Valid: true}, // The exceptions are INTEGER PRIMARY KEY
UniqueValue: sql.NullBool{Bool: index.Unique, Valid: true},
})
}
return nil
})
return indexes, err
}
func (m Migrator) getRawDDL(table string) (string, error) {
+3
View File
@@ -0,0 +1,3 @@
# This source code refers to The Go Authors for copyright purposes.
# The master list of authors is in the main Go distribution,
# visible at https://tip.golang.org/AUTHORS.
+3
View File
@@ -0,0 +1,3 @@
# This source code was written by the Go contributors.
# The master list of contributors is in the main Go distribution,
# visible at https://tip.golang.org/CONTRIBUTORS.
+27
View File
@@ -0,0 +1,27 @@
Copyright (c) 2020 The Go Authors. All rights reserved.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are
met:
* Redistributions of source code must retain the above copyright
notice, this list of conditions and the following disclaimer.
* Redistributions in binary form must reproduce the above
copyright notice, this list of conditions and the following disclaimer
in the documentation and/or other materials provided with the
distribution.
* Neither the name of Google Inc. nor the names of its
contributors may be used to endorse or promote products derived from
this software without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+14
View File
@@ -0,0 +1,14 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package internal
// NotForPublicUse is a marker type that an API is for internal use only.
// It does not perfectly prevent usage of that API, but helps to restrict usage.
// Anything with this marker is not covered by the Go compatibility agreement.
type NotForPublicUse struct{}
// AllowInternalUse is passed from "json" to "jsontext" to authenticate
// that the caller can have access to internal functionality.
var AllowInternalUse NotForPublicUse
+184
View File
@@ -0,0 +1,184 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// jsonflags implements all the optional boolean flags.
// These flags are shared across both "json", "jsontext", and "jsonopts".
package jsonflags
import "github.com/go-json-experiment/json/internal"
// Bools represents zero or more boolean flags, all set to true or false.
// The least-significant bit is the boolean value of all flags in the set.
// The remaining bits identify which particular flags.
//
// In common usage, this is OR'd with 0 or 1. For example:
// - (AllowInvalidUTF8 | 0) means "AllowInvalidUTF8 is false"
// - (Expand | Indent | 1) means "Expand and Indent are true"
type Bools uint64
func (Bools) JSONOptions(internal.NotForPublicUse) {}
const (
// AllFlags is the set of all flags.
AllFlags = AllCoderFlags | AllArshalV2Flags | AllArshalV1Flags
// AllCoderFlags is the set of all encoder/decoder flags.
AllCoderFlags = (maxCoderFlag - 1) - initFlag
// AllArshalV2Flags is the set of all v2 marshal/unmarshal flags.
AllArshalV2Flags = (maxArshalV2Flag - 1) - (maxCoderFlag - 1)
// AllArshalV1Flags is the set of all v1 marshal/unmarshal flags.
AllArshalV1Flags = (maxArshalV1Flag - 1) - (maxArshalV2Flag - 1)
// NonBooleanFlags is the set of non-boolean flags,
// where the value is some other concrete Go type.
// The value of the flag is stored within jsonopts.Struct.
NonBooleanFlags = 0 |
Indent |
IndentPrefix |
ByteLimit |
DepthLimit |
Marshalers |
Unmarshalers
// DefaultV1Flags is the set of booleans flags that default to true under
// v1 semantics. None of the non-boolean flags differ between v1 and v2.
DefaultV1Flags = 0 |
AllowDuplicateNames |
AllowInvalidUTF8 |
EscapeForHTML |
EscapeForJS |
Deterministic |
FormatNilMapAsNull |
FormatNilSliceAsNull |
MatchCaseInsensitiveNames |
FormatByteArrayAsArray |
FormatTimeDurationAsNanosecond |
IgnoreStructErrors |
MatchCaseSensitiveDelimiter |
MergeWithLegacySemantics |
OmitEmptyWithLegacyDefinition |
RejectFloatOverflow |
ReportLegacyErrorValues |
SkipUnaddressableMethods |
StringifyWithLegacySemantics |
UnmarshalArrayFromAnyLength
)
// Encoder and decoder flags.
const (
initFlag Bools = 1 << iota // reserved for the boolean value itself
AllowDuplicateNames // encode or decode
AllowInvalidUTF8 // encode or decode
WithinArshalCall // encode or decode; for internal use by json.Marshal and json.Unmarshal
OmitTopLevelNewline // encode only; for internal use by json.Marshal and json.MarshalWrite
PreserveRawStrings // encode only; for internal use by jsontext.Value.Canonicalize
CanonicalizeNumbers // encode only; for internal use by jsontext.Value.Canonicalize
EscapeForHTML // encode only
EscapeForJS // encode only
Expand // encode only
Indent // encode only; non-boolean flag
IndentPrefix // encode only; non-boolean flag
ByteLimit // encode or decode; non-boolean flag
DepthLimit // encode or decode; non-boolean flag
maxCoderFlag
)
// Marshal and Unmarshal flags (for v2).
const (
_ Bools = (maxCoderFlag >> 1) << iota
StringifyNumbers // marshal or unmarshal
Deterministic // marshal only
FormatNilMapAsNull // marshal only
FormatNilSliceAsNull // marshal only
MatchCaseInsensitiveNames // marshal or unmarshal
DiscardUnknownMembers // marshal only
RejectUnknownMembers // unmarshal only
Marshalers // marshal only; non-boolean flag
Unmarshalers // unmarshal only; non-boolean flag
maxArshalV2Flag
)
// Marshal and Unmarshal flags (for v1).
const (
_ Bools = (maxArshalV2Flag >> 1) << iota
FormatByteArrayAsArray // marshal or unmarshal
FormatTimeDurationAsNanosecond // marshal or unmarshal
IgnoreStructErrors // marshal or unmarshal
MatchCaseSensitiveDelimiter // marshal or unmarshal
MergeWithLegacySemantics // unmarshal
OmitEmptyWithLegacyDefinition // marshal
RejectFloatOverflow // unmarshal
ReportLegacyErrorValues // marshal or unmarshal
SkipUnaddressableMethods // marshal or unmarshal
StringifyWithLegacySemantics // marshal or unmarshal
UnmarshalAnyWithRawNumber // unmarshal; for internal use by jsonv1.Decoder.UseNumber
UnmarshalArrayFromAnyLength // unmarshal
maxArshalV1Flag
)
// Flags is a set of boolean flags.
// If the presence bit is zero, then the value bit must also be zero.
// The least-significant bit of both fields is always zero.
//
// Unlike Bools, which can represent a set of bools that are all true or false,
// Flags represents a set of bools, each individually may be true or false.
type Flags struct{ Presence, Values uint64 }
// Join joins two sets of flags such that the latter takes precedence.
func (dst *Flags) Join(src Flags) {
// Copy over all source presence bits over to the destination (using OR),
// then invert the source presence bits to clear out source value (using AND-NOT),
// then copy over source value bits over to the destination (using OR).
// e.g., dst := Flags{Presence: 0b_1100_0011, Value: 0b_1000_0011}
// e.g., src := Flags{Presence: 0b_0101_1010, Value: 0b_1001_0010}
dst.Presence |= src.Presence // e.g., 0b_1100_0011 | 0b_0101_1010 -> 0b_110_11011
dst.Values &= ^src.Presence // e.g., 0b_1000_0011 & 0b_1010_0101 -> 0b_100_00001
dst.Values |= src.Values // e.g., 0b_1000_0001 | 0b_1001_0010 -> 0b_100_10011
}
// Set sets both the presence and value for the provided bool (or set of bools).
func (fs *Flags) Set(f Bools) {
// Select out the bits for the flag identifiers (everything except LSB),
// then set the presence for all the identifier bits (using OR),
// then invert the identifier bits to clear out the values (using AND-NOT),
// then copy over all the identifier bits to the value if LSB is 1.
// e.g., fs := Flags{Presence: 0b_0101_0010, Value: 0b_0001_0010}
// e.g., f := 0b_1001_0001
id := uint64(f) &^ uint64(1) // e.g., 0b_1001_0001 & 0b_1111_1110 -> 0b_1001_0000
fs.Presence |= id // e.g., 0b_0101_0010 | 0b_1001_0000 -> 0b_1101_0011
fs.Values &= ^id // e.g., 0b_0001_0010 & 0b_0110_1111 -> 0b_0000_0010
fs.Values |= uint64(f&1) * id // e.g., 0b_0000_0010 | 0b_1001_0000 -> 0b_1001_0010
}
// Get reports whether the bool (or any of the bools) is true.
// This is generally only used with a singular bool.
// The value bit of f (i.e., the LSB) is ignored.
func (fs Flags) Get(f Bools) bool {
return fs.Values&uint64(f) > 0
}
// Has reports whether the bool (or any of the bools) is set.
// The value bit of f (i.e., the LSB) is ignored.
func (fs Flags) Has(f Bools) bool {
return fs.Presence&uint64(f) > 0
}
// Clear clears both the presence and value for the provided bool or bools.
// The value bit of f (i.e., the LSB) is ignored.
func (fs *Flags) Clear(f Bools) {
// Invert f to produce a mask to clear all bits in f (using AND).
// e.g., fs := Flags{Presence: 0b_0101_0010, Value: 0b_0001_0010}
// e.g., f := 0b_0001_1000
mask := uint64(^f) // e.g., 0b_0001_1000 -> 0b_1110_0111
fs.Presence &= mask // e.g., 0b_0101_0010 & 0b_1110_0111 -> 0b_0100_0010
fs.Values &= mask // e.g., 0b_0001_0010 & 0b_1110_0111 -> 0b_0000_0010
}
+190
View File
@@ -0,0 +1,190 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonopts
import (
"github.com/go-json-experiment/json/internal"
"github.com/go-json-experiment/json/internal/jsonflags"
)
// Options is the common options type shared across json packages.
type Options interface {
// JSONOptions is exported so related json packages can implement Options.
JSONOptions(internal.NotForPublicUse)
}
// Struct is the combination of all options in struct form.
// This is efficient to pass down the call stack and to query.
type Struct struct {
Flags jsonflags.Flags
CoderValues
ArshalValues
}
type CoderValues struct {
Indent string // jsonflags.Indent
IndentPrefix string // jsonflags.IndentPrefix
ByteLimit int64 // jsonflags.ByteLimit
DepthLimit int // jsonflags.DepthLimit
}
type ArshalValues struct {
// The Marshalers and Unmarshalers fields use the any type to avoid a
// concrete dependency on *json.Marshalers and *json.Unmarshalers,
// which would in turn create a dependency on the "reflect" package.
Marshalers any // jsonflags.Marshalers
Unmarshalers any // jsonflags.Unmarshalers
Format string
FormatDepth int
}
// DefaultOptionsV2 is the set of all options that define default v2 behavior.
var DefaultOptionsV2 = Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.AllFlags),
Values: uint64(0),
},
CoderValues: CoderValues{Indent: "\t"}, // Indent is set, but Expand is set to false
}
// DefaultOptionsV1 is the set of all options that define default v1 behavior.
var DefaultOptionsV1 = Struct{
Flags: jsonflags.Flags{
Presence: uint64(jsonflags.AllFlags),
Values: uint64(jsonflags.DefaultV1Flags),
},
CoderValues: CoderValues{Indent: "\t"}, // Indent is set, but Expand is set to false
}
// CopyCoderOptions copies coder-specific options from src to dst.
// This is used by json.MarshalEncode and json.UnmarshalDecode since those
// functions ignore any coder-specific options and uses the options from the
// Encoder or Decoder that is passed in.
func (dst *Struct) CopyCoderOptions(src *Struct) {
srcFlags := src.Flags
srcFlags.Clear(^jsonflags.AllCoderFlags)
dst.Flags.Join(srcFlags)
dst.CoderValues = src.CoderValues
}
func (*Struct) JSONOptions(internal.NotForPublicUse) {}
// GetUnknownOption is injected by the "json" package to handle Options
// declared in that package so that "jsonopts" can handle them.
var GetUnknownOption = func(*Struct, Options) (any, bool) { panic("unknown option") }
func GetOption[T any](opts Options, setter func(T) Options) (T, bool) {
// Collapse the options to *Struct to simplify lookup.
structOpts, ok := opts.(*Struct)
if !ok {
var structOpts2 Struct
structOpts2.Join(opts)
structOpts = &structOpts2
}
// Lookup the option based on the return value of the setter.
var zero T
switch opt := setter(zero).(type) {
case jsonflags.Bools:
v := structOpts.Flags.Get(opt)
ok := structOpts.Flags.Has(opt)
return any(v).(T), ok
case Indent:
if !structOpts.Flags.Has(jsonflags.Indent) {
return zero, false
}
return any(structOpts.Indent).(T), true
case IndentPrefix:
if !structOpts.Flags.Has(jsonflags.IndentPrefix) {
return zero, false
}
return any(structOpts.IndentPrefix).(T), true
case ByteLimit:
if !structOpts.Flags.Has(jsonflags.ByteLimit) {
return zero, false
}
return any(structOpts.ByteLimit).(T), true
case DepthLimit:
if !structOpts.Flags.Has(jsonflags.DepthLimit) {
return zero, false
}
return any(structOpts.DepthLimit).(T), true
default:
v, ok := GetUnknownOption(structOpts, opt)
return v.(T), ok
}
}
// JoinUnknownOption is injected by the "json" package to handle Options
// declared in that package so that "jsonopts" can handle them.
var JoinUnknownOption = func(*Struct, Options) { panic("unknown option") }
func (dst *Struct) Join(srcs ...Options) {
for _, src := range srcs {
switch src := src.(type) {
case nil:
continue
case jsonflags.Bools:
dst.Flags.Set(src)
case Indent:
dst.Flags.Set(jsonflags.Expand | jsonflags.Indent | 1)
dst.Indent = string(src)
case IndentPrefix:
dst.Flags.Set(jsonflags.Expand | jsonflags.IndentPrefix | 1)
dst.IndentPrefix = string(src)
case ByteLimit:
dst.Flags.Set(jsonflags.ByteLimit | 1)
dst.ByteLimit = int64(src)
case DepthLimit:
dst.Flags.Set(jsonflags.DepthLimit | 1)
dst.DepthLimit = int(src)
case *Struct:
dst.Flags.Join(src.Flags)
if src.Flags.Has(jsonflags.NonBooleanFlags) {
if src.Flags.Has(jsonflags.Indent) {
dst.Indent = src.Indent
}
if src.Flags.Has(jsonflags.IndentPrefix) {
dst.IndentPrefix = src.IndentPrefix
}
if src.Flags.Has(jsonflags.ByteLimit) {
dst.ByteLimit = src.ByteLimit
}
if src.Flags.Has(jsonflags.DepthLimit) {
dst.DepthLimit = src.DepthLimit
}
if src.Flags.Has(jsonflags.Marshalers) {
dst.Marshalers = src.Marshalers
}
if src.Flags.Has(jsonflags.Unmarshalers) {
dst.Unmarshalers = src.Unmarshalers
}
}
if src.Format != "" {
dst.Format = src.Format
dst.FormatDepth = src.FormatDepth
}
default:
JoinUnknownOption(dst, src)
}
}
}
type (
Indent string // jsontext.WithIndent
IndentPrefix string // jsontext.WithIndentPrefix
ByteLimit int64 // jsontext.WithByteLimit
DepthLimit int // jsontext.WithDepthLimit
// type for jsonflags.Marshalers declared in "json" package
// type for jsonflags.Unmarshalers declared in "json" package
)
func (Indent) JSONOptions(internal.NotForPublicUse) {}
func (IndentPrefix) JSONOptions(internal.NotForPublicUse) {}
func (ByteLimit) JSONOptions(internal.NotForPublicUse) {}
func (DepthLimit) JSONOptions(internal.NotForPublicUse) {}
+640
View File
@@ -0,0 +1,640 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonwire
import (
"io"
"math"
"slices"
"strconv"
"unicode/utf16"
"unicode/utf8"
)
type ValueFlags uint
const (
_ ValueFlags = (1 << iota) / 2 // powers of two starting with zero
stringNonVerbatim // string cannot be naively treated as valid UTF-8
stringNonCanonical // string not formatted according to RFC 8785, section 3.2.2.2.
// TODO: Track whether a number is a non-integer?
)
func (f *ValueFlags) Join(f2 ValueFlags) { *f |= f2 }
func (f ValueFlags) IsVerbatim() bool { return f&stringNonVerbatim == 0 }
func (f ValueFlags) IsCanonical() bool { return f&stringNonCanonical == 0 }
// ConsumeWhitespace consumes leading JSON whitespace per RFC 7159, section 2.
func ConsumeWhitespace(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
for len(b) > n && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') {
n++
}
return n
}
// ConsumeNull consumes the next JSON null literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
func ConsumeNull(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "null"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeFalse consumes the next JSON false literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
func ConsumeFalse(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "false"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeTrue consumes the next JSON true literal per RFC 7159, section 3.
// It returns 0 if it is invalid, in which case consumeLiteral should be used.
func ConsumeTrue(b []byte) int {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
const literal = "true"
if len(b) >= len(literal) && string(b[:len(literal)]) == literal {
return len(literal)
}
return 0
}
// ConsumeLiteral consumes the next JSON literal per RFC 7159, section 3.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
func ConsumeLiteral(b []byte, lit string) (n int, err error) {
for i := 0; i < len(b) && i < len(lit); i++ {
if b[i] != lit[i] {
return i, NewInvalidCharacterError(b[i:], "within literal "+lit+" (expecting "+strconv.QuoteRune(rune(lit[i]))+")")
}
}
if len(b) < len(lit) {
return len(b), io.ErrUnexpectedEOF
}
return len(lit), nil
}
// ConsumeSimpleString consumes the next JSON string per RFC 7159, section 7
// but is limited to the grammar for an ASCII string without escape sequences.
// It returns 0 if it is invalid or more complicated than a simple string,
// in which case consumeString should be called.
//
// It rejects '<', '>', and '&' for compatibility reasons since these were
// always escaped in the v1 implementation. Thus, if this function reports
// non-zero then we know that the string would be encoded the same way
// under both v1 or v2 escape semantics.
func ConsumeSimpleString(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[0] == '"' {
n++
for len(b) > n && b[n] < utf8.RuneSelf && escapeASCII[b[n]] == 0 {
n++
}
if uint(len(b)) > uint(n) && b[n] == '"' {
n++
return n
}
}
return 0
}
// ConsumeString consumes the next JSON string per RFC 7159, section 7.
// If validateUTF8 is false, then this allows the presence of invalid UTF-8
// characters within the string itself.
// It reports the number of bytes consumed and whether an error was encountered.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
func ConsumeString(flags *ValueFlags, b []byte, validateUTF8 bool) (n int, err error) {
return ConsumeStringResumable(flags, b, 0, validateUTF8)
}
// ConsumeStringResumable is identical to consumeString but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeStringResumable(flags *ValueFlags, b []byte, resumeOffset int, validateUTF8 bool) (n int, err error) {
// Consume the leading double quote.
switch {
case resumeOffset > 0:
n = resumeOffset // already handled the leading quote
case uint(len(b)) == 0:
return n, io.ErrUnexpectedEOF
case b[0] == '"':
n++
default:
return n, NewInvalidCharacterError(b[n:], `at start of string (expecting '"')`)
}
// Consume every character in the string.
for uint(len(b)) > uint(n) {
// Optimize for long sequences of unescaped characters.
noEscape := func(c byte) bool {
return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"'
}
for uint(len(b)) > uint(n) && noEscape(b[n]) {
n++
}
if uint(len(b)) <= uint(n) {
return n, io.ErrUnexpectedEOF
}
// Check for terminating double quote.
if b[n] == '"' {
n++
return n, nil
}
switch r, rn := utf8.DecodeRune(b[n:]); {
// Handle UTF-8 encoded byte sequence.
// Due to specialized handling of ASCII above, we know that
// all normal sequences at this point must be 2 bytes or larger.
case rn > 1:
n += rn
// Handle escape sequence.
case r == '\\':
flags.Join(stringNonVerbatim)
resumeOffset = n
if uint(len(b)) < uint(n+2) {
return resumeOffset, io.ErrUnexpectedEOF
}
switch r := b[n+1]; r {
case '/':
// Forward slash is the only character with 3 representations.
// Per RFC 8785, section 3.2.2.2., this must not be escaped.
flags.Join(stringNonCanonical)
n += 2
case '"', '\\', 'b', 'f', 'n', 'r', 't':
n += 2
case 'u':
if uint(len(b)) < uint(n+6) {
if hasEscapedUTF16Prefix(b[n:], false) {
return resumeOffset, io.ErrUnexpectedEOF
}
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n:])
}
v1, ok := parseHexUint16(b[n+2 : n+6])
if !ok {
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n : n+6])
}
// Only certain control characters can use the \uFFFF notation
// for canonical formatting (per RFC 8785, section 3.2.2.2.).
switch v1 {
// \uFFFF notation not permitted for these characters.
case '\b', '\f', '\n', '\r', '\t':
flags.Join(stringNonCanonical)
default:
// \uFFFF notation only permitted for control characters.
if v1 >= ' ' {
flags.Join(stringNonCanonical)
} else {
// \uFFFF notation must be lower case.
for _, c := range b[n+2 : n+6] {
if 'A' <= c && c <= 'F' {
flags.Join(stringNonCanonical)
}
}
}
}
n += 6
r := rune(v1)
if validateUTF8 && utf16.IsSurrogate(r) {
if uint(len(b)) < uint(n+6) {
if hasEscapedUTF16Prefix(b[n:], true) {
return resumeOffset, io.ErrUnexpectedEOF
}
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6:])
} else if v2, ok := parseHexUint16(b[n+2 : n+6]); b[n] != '\\' || b[n+1] != 'u' || !ok {
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6])
} else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError {
flags.Join(stringNonCanonical)
return n - 6, NewInvalidEscapeSequenceError(b[n-6 : n+6])
} else {
n += 6
}
}
default:
flags.Join(stringNonCanonical)
return n, NewInvalidEscapeSequenceError(b[n : n+2])
}
// Handle invalid UTF-8.
case r == utf8.RuneError:
if !utf8.FullRune(b[n:]) {
return n, io.ErrUnexpectedEOF
}
flags.Join(stringNonVerbatim | stringNonCanonical)
if validateUTF8 {
return n, ErrInvalidUTF8
}
n++
// Handle invalid control characters.
case r < ' ':
flags.Join(stringNonVerbatim | stringNonCanonical)
return n, NewInvalidCharacterError(b[n:], "within string (expecting non-control character)")
default:
panic("BUG: unhandled character " + QuoteRune(b[n:]))
}
}
return n, io.ErrUnexpectedEOF
}
// AppendUnquote appends the unescaped form of a JSON string in src to dst.
// Any invalid UTF-8 within the string will be replaced with utf8.RuneError,
// but the error will be specified as having encountered such an error.
// The input must be an entire JSON string with no surrounding whitespace.
func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) (v []byte, err error) {
dst = slices.Grow(dst, len(src))
// Consume the leading double quote.
var i, n int
switch {
case uint(len(src)) == 0:
return dst, io.ErrUnexpectedEOF
case src[0] == '"':
i, n = 1, 1
default:
return dst, NewInvalidCharacterError(src, `at start of string (expecting '"')`)
}
// Consume every character in the string.
for uint(len(src)) > uint(n) {
// Optimize for long sequences of unescaped characters.
noEscape := func(c byte) bool {
return c < utf8.RuneSelf && ' ' <= c && c != '\\' && c != '"'
}
for uint(len(src)) > uint(n) && noEscape(src[n]) {
n++
}
if uint(len(src)) <= uint(n) {
dst = append(dst, src[i:n]...)
return dst, io.ErrUnexpectedEOF
}
// Check for terminating double quote.
if src[n] == '"' {
dst = append(dst, src[i:n]...)
n++
if n < len(src) {
err = NewInvalidCharacterError(src[n:], "after string value")
}
return dst, err
}
switch r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:]))); {
// Handle UTF-8 encoded byte sequence.
// Due to specialized handling of ASCII above, we know that
// all normal sequences at this point must be 2 bytes or larger.
case rn > 1:
n += rn
// Handle escape sequence.
case r == '\\':
dst = append(dst, src[i:n]...)
// Handle escape sequence.
if uint(len(src)) < uint(n+2) {
return dst, io.ErrUnexpectedEOF
}
switch r := src[n+1]; r {
case '"', '\\', '/':
dst = append(dst, r)
n += 2
case 'b':
dst = append(dst, '\b')
n += 2
case 'f':
dst = append(dst, '\f')
n += 2
case 'n':
dst = append(dst, '\n')
n += 2
case 'r':
dst = append(dst, '\r')
n += 2
case 't':
dst = append(dst, '\t')
n += 2
case 'u':
if uint(len(src)) < uint(n+6) {
if hasEscapedUTF16Prefix(src[n:], false) {
return dst, io.ErrUnexpectedEOF
}
return dst, NewInvalidEscapeSequenceError(src[n:])
}
v1, ok := parseHexUint16(src[n+2 : n+6])
if !ok {
return dst, NewInvalidEscapeSequenceError(src[n : n+6])
}
n += 6
// Check whether this is a surrogate half.
r := rune(v1)
if utf16.IsSurrogate(r) {
r = utf8.RuneError // assume failure unless the following succeeds
if uint(len(src)) < uint(n+6) {
if hasEscapedUTF16Prefix(src[n:], true) {
return utf8.AppendRune(dst, r), io.ErrUnexpectedEOF
}
err = NewInvalidEscapeSequenceError(src[n-6:])
} else if v2, ok := parseHexUint16(src[n+2 : n+6]); src[n] != '\\' || src[n+1] != 'u' || !ok {
err = NewInvalidEscapeSequenceError(src[n-6 : n+6])
} else if r = utf16.DecodeRune(rune(v1), rune(v2)); r == utf8.RuneError {
err = NewInvalidEscapeSequenceError(src[n-6 : n+6])
} else {
n += 6
}
}
dst = utf8.AppendRune(dst, r)
default:
return dst, NewInvalidEscapeSequenceError(src[n : n+2])
}
i = n
// Handle invalid UTF-8.
case r == utf8.RuneError:
dst = append(dst, src[i:n]...)
if !utf8.FullRuneInString(string(truncateMaxUTF8(src[n:]))) {
return dst, io.ErrUnexpectedEOF
}
// NOTE: An unescaped string may be longer than the escaped string
// because invalid UTF-8 bytes are being replaced.
dst = append(dst, "\uFFFD"...)
n += rn
i = n
err = ErrInvalidUTF8
// Handle invalid control characters.
case r < ' ':
dst = append(dst, src[i:n]...)
return dst, NewInvalidCharacterError(src[n:], "within string (expecting non-control character)")
default:
panic("BUG: unhandled character " + QuoteRune(src[n:]))
}
}
dst = append(dst, src[i:n]...)
return dst, io.ErrUnexpectedEOF
}
// hasEscapedUTF16Prefix reports whether b is possibly
// the truncated prefix of a \uFFFF escape sequence.
func hasEscapedUTF16Prefix[Bytes ~[]byte | ~string](b Bytes, lowerSurrogateHalf bool) bool {
for i := 0; i < len(b); i++ {
switch c := b[i]; {
case i == 0 && c != '\\':
return false
case i == 1 && c != 'u':
return false
case i == 2 && lowerSurrogateHalf && c != 'd' && c != 'D':
return false // not within ['\uDC00':'\uDFFF']
case i == 3 && lowerSurrogateHalf && !('c' <= c && c <= 'f') && !('C' <= c && c <= 'F'):
return false // not within ['\uDC00':'\uDFFF']
case i >= 2 && i < 6 && !('0' <= c && c <= '9') && !('a' <= c && c <= 'f') && !('A' <= c && c <= 'F'):
return false
}
}
return true
}
// UnquoteMayCopy returns the unescaped form of b.
// If there are no escaped characters, the output is simply a subslice of
// the input with the surrounding quotes removed.
// Otherwise, a new buffer is allocated for the output.
// It assumes the input is valid.
func UnquoteMayCopy(b []byte, isVerbatim bool) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if isVerbatim {
return b[len(`"`) : len(b)-len(`"`)]
}
b, _ = AppendUnquote(nil, b)
return b
}
// ConsumeSimpleNumber consumes the next JSON number per RFC 7159, section 6
// but is limited to the grammar for a positive integer.
// It returns 0 if it is invalid or more complicated than a simple integer,
// in which case consumeNumber should be called.
func ConsumeSimpleNumber(b []byte) (n int) {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 {
if b[0] == '0' {
n++
} else if '1' <= b[0] && b[0] <= '9' {
n++
for len(b) > n && ('0' <= b[n] && b[n] <= '9') {
n++
}
} else {
return 0
}
if uint(len(b)) <= uint(n) || (b[n] != '.' && b[n] != 'e' && b[n] != 'E') {
return n
}
}
return 0
}
type ConsumeNumberState uint
const (
consumeNumberInit ConsumeNumberState = iota
beforeIntegerDigits
withinIntegerDigits
beforeFractionalDigits
withinFractionalDigits
beforeExponentDigits
withinExponentDigits
)
// ConsumeNumber consumes the next JSON number per RFC 7159, section 6.
// It reports the number of bytes consumed and whether an error was encountered.
// If the input appears truncated, it returns io.ErrUnexpectedEOF.
//
// Note that JSON numbers are not self-terminating.
// If the entire input is consumed, then the caller needs to consider whether
// there may be subsequent unread data that may still be part of this number.
func ConsumeNumber(b []byte) (n int, err error) {
n, _, err = ConsumeNumberResumable(b, 0, consumeNumberInit)
return n, err
}
// ConsumeNumberResumable is identical to consumeNumber but supports resuming
// from a previous call that returned io.ErrUnexpectedEOF.
func ConsumeNumberResumable(b []byte, resumeOffset int, state ConsumeNumberState) (n int, _ ConsumeNumberState, err error) {
// Jump to the right state when resuming from a partial consumption.
n = resumeOffset
if state > consumeNumberInit {
switch state {
case withinIntegerDigits, withinFractionalDigits, withinExponentDigits:
// Consume leading digits.
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
if uint(len(b)) <= uint(n) {
return n, state, nil // still within the same state
}
state++ // switches "withinX" to "beforeY" where Y is the state after X
}
switch state {
case beforeIntegerDigits:
goto beforeInteger
case beforeFractionalDigits:
goto beforeFractional
case beforeExponentDigits:
goto beforeExponent
default:
return n, state, nil
}
}
// Consume required integer component (with optional minus sign).
beforeInteger:
resumeOffset = n
if uint(len(b)) > 0 && b[0] == '-' {
n++
}
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeIntegerDigits, io.ErrUnexpectedEOF
case b[n] == '0':
n++
state = beforeFractionalDigits
case '1' <= b[n] && b[n] <= '9':
n++
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinIntegerDigits
default:
return n, state, NewInvalidCharacterError(b[n:], "within number (expecting digit)")
}
// Consume optional fractional component.
beforeFractional:
if uint(len(b)) > uint(n) && b[n] == '.' {
resumeOffset = n
n++
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeFractionalDigits, io.ErrUnexpectedEOF
case '0' <= b[n] && b[n] <= '9':
n++
default:
return n, state, NewInvalidCharacterError(b[n:], "within number (expecting digit)")
}
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinFractionalDigits
}
// Consume optional exponent component.
beforeExponent:
if uint(len(b)) > uint(n) && (b[n] == 'e' || b[n] == 'E') {
resumeOffset = n
n++
if uint(len(b)) > uint(n) && (b[n] == '-' || b[n] == '+') {
n++
}
switch {
case uint(len(b)) <= uint(n):
return resumeOffset, beforeExponentDigits, io.ErrUnexpectedEOF
case '0' <= b[n] && b[n] <= '9':
n++
default:
return n, state, NewInvalidCharacterError(b[n:], "within number (expecting digit)")
}
for uint(len(b)) > uint(n) && ('0' <= b[n] && b[n] <= '9') {
n++
}
state = withinExponentDigits
}
return n, state, nil
}
// parseHexUint16 is similar to strconv.ParseUint,
// but operates directly on []byte and is optimized for base-16.
// See https://go.dev/issue/42429.
func parseHexUint16[Bytes ~[]byte | ~string](b Bytes) (v uint16, ok bool) {
if len(b) != 4 {
return 0, false
}
for i := 0; i < 4; i++ {
c := b[i]
switch {
case '0' <= c && c <= '9':
c = c - '0'
case 'a' <= c && c <= 'f':
c = 10 + c - 'a'
case 'A' <= c && c <= 'F':
c = 10 + c - 'A'
default:
return 0, false
}
v = v*16 + uint16(c)
}
return v, true
}
// ParseUint parses b as a decimal unsigned integer according to
// a strict subset of the JSON number grammar, returning the value if valid.
// It returns (0, false) if there is a syntax error and
// returns (math.MaxUint64, false) if there is an overflow.
func ParseUint(b []byte) (v uint64, ok bool) {
const unsafeWidth = 20 // len(fmt.Sprint(uint64(math.MaxUint64)))
var n int
for ; len(b) > n && ('0' <= b[n] && b[n] <= '9'); n++ {
v = 10*v + uint64(b[n]-'0')
}
switch {
case n == 0 || len(b) != n || (b[0] == '0' && string(b) != "0"):
return 0, false
case n >= unsafeWidth && (b[0] != '1' || v < 1e19 || n > unsafeWidth):
return math.MaxUint64, false
}
return v, true
}
// ParseFloat parses a floating point number according to the Go float grammar.
// Note that the JSON number grammar is a strict subset.
//
// If the number overflows the finite representation of a float,
// then we return MaxFloat since any finite value will always be infinitely
// more accurate at representing another finite value than an infinite value.
func ParseFloat(b []byte, bits int) (v float64, ok bool) {
// Fast path for exact integer numbers which fit in the
// 24-bit or 53-bit significand of a float32 or float64.
var negLen int // either 0 or 1
if len(b) > 0 && b[0] == '-' {
negLen = 1
}
u, ok := ParseUint(b[negLen:])
if ok && ((bits == 32 && u <= 1<<24) || (bits == 64 && u <= 1<<53)) {
return math.Copysign(float64(u), float64(-1*negLen)), true
}
// Note that the []byte->string conversion unfortunately allocates.
// See https://go.dev/issue/42429 for more information.
fv, err := strconv.ParseFloat(string(b), bits)
if math.IsInf(fv, 0) {
switch {
case bits == 32 && math.IsInf(fv, +1):
fv = +math.MaxFloat32
case bits == 64 && math.IsInf(fv, +1):
fv = +math.MaxFloat64
case bits == 32 && math.IsInf(fv, -1):
fv = -math.MaxFloat32
case bits == 64 && math.IsInf(fv, -1):
fv = -math.MaxFloat64
}
}
return fv, err == nil
}
+234
View File
@@ -0,0 +1,234 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsonwire
import (
"math"
"slices"
"strconv"
"unicode/utf16"
"unicode/utf8"
"github.com/go-json-experiment/json/internal/jsonflags"
)
// escapeASCII reports whether the ASCII character needs to be escaped.
// It conservatively assumes EscapeForHTML.
var escapeASCII = [...]uint8{
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters
1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, // escape control characters
0, 0, 1, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, // escape '"' and '&'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 1, 0, // escape '<' and '>'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, // escape '\\'
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
}
// NeedEscape reports whether src needs escaping of any characters.
// It conservatively assumes EscapeForHTML and EscapeForJS.
// It reports true for inputs with invalid UTF-8.
func NeedEscape[Bytes ~[]byte | ~string](src Bytes) bool {
var i int
for uint(len(src)) > uint(i) {
if c := src[i]; c < utf8.RuneSelf {
if escapeASCII[c] > 0 {
return true
}
i++
} else {
r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[i:])))
if r == utf8.RuneError || r == '\u2028' || r == '\u2029' {
return true
}
i += rn
}
}
return false
}
// AppendQuote appends src to dst as a JSON string per RFC 7159, section 7.
//
// It takes in flags and respects the following:
// - EscapeForHTML escapes '<', '>', and '&'.
// - EscapeForJS escapes '\u2028' and '\u2029'.
// - AllowInvalidUTF8 avoids reporting an error for invalid UTF-8.
//
// Regardless of whether AllowInvalidUTF8 is specified,
// invalid bytes are replaced with the Unicode replacement character ('\ufffd').
// If no escape flags are set, then the shortest representable form is used,
// which is also the canonical form for strings (RFC 8785, section 3.2.2.2).
func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes, flags *jsonflags.Flags) ([]byte, error) {
var i, n int
var hasInvalidUTF8 bool
dst = slices.Grow(dst, len(`"`)+len(src)+len(`"`))
dst = append(dst, '"')
for uint(len(src)) > uint(n) {
// Handle single-byte ASCII.
if c := src[n]; c < utf8.RuneSelf {
n++
if escapeASCII[c] > 0 {
if (c == '<' || c == '>' || c == '&') && !flags.Get(jsonflags.EscapeForHTML) {
continue
}
dst = append(dst, src[i:n-1]...)
dst = appendEscapedASCII(dst, c)
i = n
}
continue
}
// Handle multi-byte Unicode.
switch r, rn := utf8.DecodeRuneInString(string(truncateMaxUTF8(src[n:]))); {
case r == utf8.RuneError && rn == 1:
hasInvalidUTF8 = true
dst = append(dst, src[i:n]...)
dst = append(dst, "\ufffd"...)
n += rn
i = n
case (r == '\u2028' || r == '\u2029') && flags.Get(jsonflags.EscapeForJS):
dst = append(dst, src[i:n]...)
dst = appendEscapedUnicode(dst, r)
n += rn
i = n
default:
n += rn
}
}
dst = append(dst, src[i:n]...)
dst = append(dst, '"')
if hasInvalidUTF8 && !flags.Get(jsonflags.AllowInvalidUTF8) {
return dst, ErrInvalidUTF8
}
return dst, nil
}
func appendEscapedASCII(dst []byte, c byte) []byte {
switch c {
case '"', '\\':
dst = append(dst, '\\', c)
case '\b':
dst = append(dst, "\\b"...)
case '\f':
dst = append(dst, "\\f"...)
case '\n':
dst = append(dst, "\\n"...)
case '\r':
dst = append(dst, "\\r"...)
case '\t':
dst = append(dst, "\\t"...)
default:
dst = appendEscapedUTF16(dst, uint16(c))
}
return dst
}
func appendEscapedUnicode(dst []byte, r rune) []byte {
if r1, r2 := utf16.EncodeRune(r); r1 != '\ufffd' && r2 != '\ufffd' {
dst = appendEscapedUTF16(dst, uint16(r1))
dst = appendEscapedUTF16(dst, uint16(r2))
} else {
dst = appendEscapedUTF16(dst, uint16(r))
}
return dst
}
func appendEscapedUTF16(dst []byte, x uint16) []byte {
const hex = "0123456789abcdef"
return append(dst, '\\', 'u', hex[(x>>12)&0xf], hex[(x>>8)&0xf], hex[(x>>4)&0xf], hex[(x>>0)&0xf])
}
// ReformatString consumes a JSON string from src and appends it to dst,
// reformatting it if necessary for the given escapeRune parameter.
// It returns the appended output and the number of consumed input bytes.
func ReformatString(dst, src []byte, flags *jsonflags.Flags) ([]byte, int, error) {
// TODO: Should this update ValueFlags as input?
var valFlags ValueFlags
n, err := ConsumeString(&valFlags, src, !flags.Get(jsonflags.AllowInvalidUTF8))
if err != nil {
return dst, n, err
}
isCanonical := !flags.Get(jsonflags.EscapeForHTML | jsonflags.EscapeForJS)
if flags.Get(jsonflags.PreserveRawStrings) || (isCanonical && valFlags.IsCanonical()) {
dst = append(dst, src[:n]...) // copy the string verbatim
return dst, n, nil
}
// TODO: Implement a direct, raw-to-raw reformat for strings.
// If the escapeRune option would have resulted in no changes to the output,
// it would be faster to simply append src to dst without going through
// an intermediary representation in a separate buffer.
b, _ := AppendUnquote(nil, src[:n])
dst, _ = AppendQuote(dst, string(b), flags)
return dst, n, nil
}
// AppendFloat appends src to dst as a JSON number per RFC 7159, section 6.
// It formats numbers similar to the ES6 number-to-string conversion.
// See https://go.dev/issue/14135.
//
// The output is identical to ECMA-262, 6th edition, section 7.1.12.1 and with
// RFC 8785, section 3.2.2.3 for 64-bit floating-point numbers except for -0,
// which is formatted as -0 instead of just 0.
//
// For 32-bit floating-point numbers,
// the output is a 32-bit equivalent of the algorithm.
// Note that ECMA-262 specifies no algorithm for 32-bit numbers.
func AppendFloat(dst []byte, src float64, bits int) []byte {
if bits == 32 {
src = float64(float32(src))
}
abs := math.Abs(src)
fmt := byte('f')
if abs != 0 {
if bits == 64 && (float64(abs) < 1e-6 || float64(abs) >= 1e21) ||
bits == 32 && (float32(abs) < 1e-6 || float32(abs) >= 1e21) {
fmt = 'e'
}
}
dst = strconv.AppendFloat(dst, src, fmt, -1, bits)
if fmt == 'e' {
// Clean up e-09 to e-9.
n := len(dst)
if n >= 4 && dst[n-4] == 'e' && dst[n-3] == '-' && dst[n-2] == '0' {
dst[n-2] = dst[n-1]
dst = dst[:n-1]
}
}
return dst
}
// ReformatNumber consumes a JSON string from src and appends it to dst,
// canonicalizing it if specified.
// It returns the appended output and the number of consumed input bytes.
func ReformatNumber(dst, src []byte, canonicalize bool) ([]byte, int, error) {
n, err := ConsumeNumber(src)
if err != nil {
return dst, n, err
}
if !canonicalize {
dst = append(dst, src[:n]...) // copy the number verbatim
return dst, n, nil
}
// Canonicalize the number per RFC 8785, section 3.2.2.3.
// As an optimization, we can copy integer numbers below 2⁵³ verbatim.
const maxExactIntegerDigits = 16 // len(strconv.AppendUint(nil, 1<<53, 10))
if n < maxExactIntegerDigits && ConsumeSimpleNumber(src[:n]) == n {
dst = append(dst, src[:n]...) // copy the number verbatim
return dst, n, nil
}
fv, _ := strconv.ParseFloat(string(src[:n]), 64)
switch {
case fv == 0:
fv = 0 // normalize negative zero as just zero
case math.IsInf(fv, +1):
fv = +math.MaxFloat64
case math.IsInf(fv, -1):
fv = -math.MaxFloat64
}
return AppendFloat(dst, fv, 64), n, nil
}
+169
View File
@@ -0,0 +1,169 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package jsonwire implements stateless functionality for handling JSON text.
package jsonwire
import (
"cmp"
"errors"
"strconv"
"strings"
"unicode"
"unicode/utf16"
"unicode/utf8"
)
// TrimSuffixWhitespace trims JSON from the end of b.
func TrimSuffixWhitespace(b []byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
n := len(b) - 1
for n >= 0 && (b[n] == ' ' || b[n] == '\t' || b[n] == '\r' || b[n] == '\n') {
n--
}
return b[:n+1]
}
// TrimSuffixString trims a valid JSON string at the end of b.
// The behavior is undefined if there is not a valid JSON string present.
func TrimSuffixString(b []byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[len(b)-1] == '"' {
b = b[:len(b)-1]
}
for len(b) >= 2 && !(b[len(b)-1] == '"' && b[len(b)-2] != '\\') {
b = b[:len(b)-1] // trim all characters except an unescaped quote
}
if len(b) > 0 && b[len(b)-1] == '"' {
b = b[:len(b)-1]
}
return b
}
// HasSuffixByte reports whether b ends with c.
func HasSuffixByte(b []byte, c byte) bool {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
return len(b) > 0 && b[len(b)-1] == c
}
// TrimSuffixByte removes c from the end of b if it is present.
func TrimSuffixByte(b []byte, c byte) []byte {
// NOTE: The arguments and logic are kept simple to keep this inlinable.
if len(b) > 0 && b[len(b)-1] == c {
return b[:len(b)-1]
}
return b
}
// QuoteRune quotes the first rune in the input.
func QuoteRune[Bytes ~[]byte | ~string](b Bytes) string {
r, n := utf8.DecodeRuneInString(string(truncateMaxUTF8(b)))
if r == utf8.RuneError && n == 1 {
return `'\x` + strconv.FormatUint(uint64(b[0]), 16) + `'`
}
return strconv.QuoteRune(r)
}
// CompareUTF16 lexicographically compares x to y according
// to the UTF-16 codepoints of the UTF-8 encoded input strings.
// This implements the ordering specified in RFC 8785, section 3.2.3.
func CompareUTF16[Bytes ~[]byte | ~string](x, y Bytes) int {
// NOTE: This is an optimized, mostly allocation-free implementation
// of CompareUTF16Simple in wire_test.go. FuzzCompareUTF16 verifies that the
// two implementations agree on the result of comparing any two strings.
isUTF16Self := func(r rune) bool {
return ('\u0000' <= r && r <= '\uD7FF') || ('\uE000' <= r && r <= '\uFFFF')
}
var invalidUTF8 bool
x0, y0 := x, y
for {
if len(x) == 0 || len(y) == 0 {
if len(x) == len(y) && invalidUTF8 {
return strings.Compare(string(x0), string(y0))
}
return cmp.Compare(len(x), len(y))
}
// ASCII fast-path.
if x[0] < utf8.RuneSelf || y[0] < utf8.RuneSelf {
if x[0] != y[0] {
return cmp.Compare(x[0], y[0])
}
x, y = x[1:], y[1:]
continue
}
// Decode next pair of runes as UTF-8.
rx, nx := utf8.DecodeRuneInString(string(truncateMaxUTF8(x)))
ry, ny := utf8.DecodeRuneInString(string(truncateMaxUTF8(y)))
selfx := isUTF16Self(rx)
selfy := isUTF16Self(ry)
switch {
// The x rune is a single UTF-16 codepoint, while
// the y rune is a surrogate pair of UTF-16 codepoints.
case selfx && !selfy:
ry, _ = utf16.EncodeRune(ry)
// The y rune is a single UTF-16 codepoint, while
// the x rune is a surrogate pair of UTF-16 codepoints.
case selfy && !selfx:
rx, _ = utf16.EncodeRune(rx)
}
if rx != ry {
return cmp.Compare(rx, ry)
}
invalidUTF8 = invalidUTF8 || (rx == utf8.RuneError && nx == 1) || (ry == utf8.RuneError && ny == 1)
x, y = x[nx:], y[ny:]
}
}
// truncateMaxUTF8 truncates b such it contains at least one rune.
//
// The utf8 package currently lacks generic variants, which complicates
// generic functions that operates on either []byte or string.
// As a hack, we always call the utf8 function operating on strings,
// but always truncate the input such that the result is identical.
//
// Example usage:
//
// utf8.DecodeRuneInString(string(truncateMaxUTF8(b)))
//
// Converting a []byte to a string is stack allocated since
// truncateMaxUTF8 guarantees that the []byte is short.
func truncateMaxUTF8[Bytes ~[]byte | ~string](b Bytes) Bytes {
// TODO(https://go.dev/issue/56948): Remove this function and
// instead directly call generic utf8 functions wherever used.
if len(b) > utf8.UTFMax {
return b[:utf8.UTFMax]
}
return b
}
// NewError and ErrInvalidUTF8 are injected by the "jsontext" package,
// so that these error types use the jsontext.SyntacticError type.
var (
NewError = errors.New
ErrInvalidUTF8 = errors.New("invalid UTF-8 within string")
)
func NewInvalidCharacterError[Bytes ~[]byte | ~string](prefix Bytes, where string) error {
what := QuoteRune(prefix)
return NewError("invalid character " + what + " " + where)
}
func NewInvalidEscapeSequenceError[Bytes ~[]byte | ~string](what Bytes) error {
label := "escape sequence"
if len(what) > 6 {
label = "surrogate pair"
}
needEscape := strings.IndexFunc(string(what), func(r rune) bool {
return r == '`' || r == utf8.RuneError || unicode.IsSpace(r) || !unicode.IsPrint(r)
}) >= 0
if needEscape {
return NewError("invalid " + label + " " + strconv.Quote(string(what)) + " within string")
} else {
return NewError("invalid " + label + " `" + string(what) + "` within string")
}
}
File diff suppressed because it is too large. Load diff
+105
View File
@@ -0,0 +1,105 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
// Package jsontext implements syntactic processing of JSON
// as specified in RFC 4627, RFC 7159, RFC 7493, RFC 8259, and RFC 8785.
// JSON is a simple data interchange format that can represent
// primitive data types such as booleans, strings, and numbers,
// in addition to structured data types such as objects and arrays.
//
// The [Encoder] and [Decoder] types are used to encode or decode
// a stream of JSON tokens or values.
//
// # Tokens and Values
//
// A JSON token refers to the basic structural elements of JSON:
//
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a start or end delimiter for a JSON object (i.e., '{' or '}')
// - a start or end delimiter for a JSON array (i.e., '[' or ']')
//
// A JSON token is represented by the [Token] type in Go. Technically,
// there are two additional structural characters (i.e., ':' and ','),
// but there is no [Token] representation for them since their presence
// can be inferred by the structure of the JSON grammar itself.
// For example, there must always be an implicit colon between
// the name and value of a JSON object member.
//
// A JSON value refers to a complete unit of JSON data:
//
// - a JSON literal, string, or number
// - a JSON object (e.g., `{"name":"value"}`)
// - a JSON array (e.g., `[1,2,3,]`)
//
// A JSON value is represented by the [Value] type in Go and is a []byte
// containing the raw textual representation of the value. There is some overlap
// between tokens and values as both contain literals, strings, and numbers.
// However, only a value can represent the entirety of a JSON object or array.
//
// The [Encoder] and [Decoder] types contain methods to read or write the next
// [Token] or [Value] in a sequence. They maintain a state machine to validate
// whether the sequence of JSON tokens and/or values produces a valid JSON.
// [Options] may be passed to the [NewEncoder] or [NewDecoder] constructors
// to configure the syntactic behavior of encoding and decoding.
//
// # Terminology
//
// The terms "encode" and "decode" are used for syntactic functionality
// that is concerned with processing JSON based on its grammar, and
// the terms "marshal" and "unmarshal" are used for semantic functionality
// that determines the meaning of JSON values as Go values and vice-versa.
// This package (i.e., [jsontext]) deals with JSON at a syntactic layer,
// while [encoding/json/v2] deals with JSON at a semantic layer.
// The goal is to provide a clear distinction between functionality that
// is purely concerned with encoding versus that of marshaling.
// For example, one can directly encode a stream of JSON tokens without
// needing to marshal a concrete Go value representing them.
// Similarly, one can decode a stream of JSON tokens without
// needing to unmarshal them into a concrete Go value.
//
// This package uses JSON terminology when discussing JSON, which may differ
// from related concepts in Go or elsewhere in computing literature.
//
// - a JSON "object" refers to an unordered collection of name/value members.
// - a JSON "array" refers to an ordered sequence of elements.
// - a JSON "value" refers to either a literal (i.e., null, false, or true),
// string, number, object, or array.
//
// See RFC 8259 for more information.
//
// # Specifications
//
// Relevant specifications include RFC 4627, RFC 7159, RFC 7493, RFC 8259,
// and RFC 8785. Each RFC is generally a stricter subset of another RFC.
// In increasing order of strictness:
//
// - RFC 4627 and RFC 7159 do not require (but recommend) the use of UTF-8
// and also do not require (but recommend) that object names be unique.
// - RFC 8259 requires the use of UTF-8,
// but does not require (but recommends) that object names be unique.
// - RFC 7493 requires the use of UTF-8
// and also requires that object names be unique.
// - RFC 8785 defines a canonical representation. It requires the use of UTF-8
// and also requires that object names be unique and in a specific ordering.
// It specifies exactly how strings and numbers must be formatted.
//
// The primary difference between RFC 4627 and RFC 7159 is that the former
// restricted top-level values to only JSON objects and arrays, while
// RFC 7159 and subsequent RFCs permit top-level values to additionally be
// JSON nulls, booleans, strings, or numbers.
//
// By default, this package operates on RFC 7493, but can be configured
// to operate according to the other RFC specifications.
// RFC 7493 is a stricter subset of RFC 8259 and fully compliant with it.
// In particular, it makes specific choices about behavior that RFC 8259
// leaves as undefined in order to ensure greater interoperability.
package jsontext
// requireKeyedLiterals can be embedded in a struct to require keyed literals.
type requireKeyedLiterals struct{}
// nonComparable can be embedded in a struct to prevent comparability.
type nonComparable [0]func()
+900
View File
@@ -0,0 +1,900 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"bytes"
"io"
"math/bits"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// Encoder is a streaming encoder from raw JSON tokens and values.
// It is used to write a stream of top-level JSON values,
// each terminated with a newline character.
//
// [Encoder.WriteToken] and [Encoder.WriteValue] calls may be interleaved.
// For example, the following JSON value:
//
// {"name":"value","array":[null,false,true,3.14159],"object":{"k":"v"}}
//
// can be composed with the following calls (ignoring errors for brevity):
//
// e.WriteToken(ObjectStart) // {
// e.WriteToken(String("name")) // "name"
// e.WriteToken(String("value")) // "value"
// e.WriteValue(Value(`"array"`)) // "array"
// e.WriteToken(ArrayStart) // [
// e.WriteToken(Null) // null
// e.WriteToken(False) // false
// e.WriteValue(Value("true")) // true
// e.WriteToken(Float(3.14159)) // 3.14159
// e.WriteToken(ArrayEnd) // ]
// e.WriteValue(Value(`"object"`)) // "object"
// e.WriteValue(Value(`{"k":"v"}`)) // {"k":"v"}
// e.WriteToken(ObjectEnd) // }
//
// The above is one of many possible sequence of calls and
// may not represent the most sensible method to call for any given token/value.
// For example, it is probably more common to call [Encoder.WriteToken] with a string
// for object names.
type Encoder struct {
s encoderState
}
// encoderState is the low-level state of Encoder.
// It has exported fields and method for use by the "json" package.
type encoderState struct {
state
encodeBuffer
jsonopts.Struct
SeenPointers map[any]struct{} // only used when marshaling; identical to json.seenPointers
}
// encodeBuffer is a buffer split into 2 segments:
//
// - buf[0:len(buf)] // written (but unflushed) portion of the buffer
// - buf[len(buf):cap(buf)] // unused portion of the buffer
type encodeBuffer struct {
Buf []byte // may alias wr if it is a bytes.Buffer
// baseOffset is added to len(buf) to obtain the absolute offset
// relative to the start of io.Writer stream.
baseOffset int64
wr io.Writer
// maxValue is the approximate maximum Value size passed to WriteValue.
maxValue int
// unusedCache is the buffer returned by the UnusedBuffer method.
unusedCache []byte
// bufStats is statistics about buffer utilization.
// It is only used with pooled encoders in pools.go.
bufStats bufferStatistics
}
// NewEncoder constructs a new streaming encoder writing to w
// configured with the provided options.
// It flushes the internal buffer when the buffer is sufficiently full or
// when a top-level value has been written.
//
// If w is a [bytes.Buffer], then the encoder appends directly into the buffer
// without copying the contents from an intermediate buffer.
func NewEncoder(w io.Writer, opts ...Options) *Encoder {
e := new(Encoder)
e.Reset(w, opts...)
return e
}
// Reset resets an encoder such that it is writing afresh to w and
// configured with the provided options. Reset must not be called on
// a Encoder passed to the [encoding/json/v2.MarshalerV2.MarshalJSONV2] method
// or the [encoding/json/v2.MarshalFuncV2] function.
func (e *Encoder) Reset(w io.Writer, opts ...Options) {
switch {
case e == nil:
panic("jsontext: invalid nil Encoder")
case w == nil:
panic("jsontext: invalid nil io.Writer")
case e.s.Flags.Get(jsonflags.WithinArshalCall):
panic("jsontext: cannot reset Encoder passed to json.MarshalerV2")
}
e.s.reset(nil, w, opts...)
}
func (e *encoderState) reset(b []byte, w io.Writer, opts ...Options) {
e.state.reset()
e.encodeBuffer = encodeBuffer{Buf: b, wr: w, bufStats: e.bufStats}
if bb, ok := w.(*bytes.Buffer); ok && bb != nil {
e.Buf = bb.Bytes()[bb.Len():] // alias the unused buffer of bb
}
e.Struct = jsonopts.Struct{}
e.Struct.Join(opts...)
if e.Flags.Get(jsonflags.Expand) && !e.Flags.Has(jsonflags.Indent) {
e.Indent = "\t"
}
}
// NeedFlush determines whether to flush at this point.
func (e *encoderState) NeedFlush() bool {
// NOTE: This function is carefully written to be inlinable.
// Avoid flushing if e.wr is nil since there is no underlying writer.
// Flush if less than 25% of the capacity remains.
// Flushing at some constant fraction ensures that the buffer stops growing
// so long as the largest Token or Value fits within that unused capacity.
return e.wr != nil && (e.Tokens.Depth() == 1 || len(e.Buf) > 3*cap(e.Buf)/4)
}
// Flush flushes the buffer to the underlying io.Writer.
// It may append a trailing newline after the top-level value.
func (e *encoderState) Flush() error {
if e.wr == nil || e.avoidFlush() {
return nil
}
// In streaming mode, always emit a newline after the top-level value.
if e.Tokens.Depth() == 1 && !e.Flags.Get(jsonflags.OmitTopLevelNewline) {
e.Buf = append(e.Buf, '\n')
}
// Inform objectNameStack that we are about to flush the buffer content.
e.Names.copyQuotedBuffer(e.Buf)
// Specialize bytes.Buffer for better performance.
if bb, ok := e.wr.(*bytes.Buffer); ok {
// If e.buf already aliases the internal buffer of bb,
// then the Write call simply increments the internal offset,
// otherwise Write operates as expected.
// See https://go.dev/issue/42986.
n, _ := bb.Write(e.Buf) // never fails unless bb is nil
e.baseOffset += int64(n)
// If the internal buffer of bytes.Buffer is too small,
// append operations elsewhere in the Encoder may grow the buffer.
// This would be semantically correct, but hurts performance.
// As such, ensure 25% of the current length is always available
// to reduce the probability that other appends must allocate.
if avail := bb.Available(); avail < bb.Len()/4 {
bb.Grow(avail + 1)
}
e.Buf = bb.AvailableBuffer()
return nil
}
// Flush the internal buffer to the underlying io.Writer.
n, err := e.wr.Write(e.Buf)
e.baseOffset += int64(n)
if err != nil {
// In the event of an error, preserve the unflushed portion.
// Thus, write errors aren't fatal so long as the io.Writer
// maintains consistent state after errors.
if n > 0 {
e.Buf = e.Buf[:copy(e.Buf, e.Buf[n:])]
}
return &ioError{action: "write", err: err}
}
e.Buf = e.Buf[:0]
// Check whether to grow the buffer.
// Note that cap(e.buf) may already exceed maxBufferSize since
// an append elsewhere already grew it to store a large token.
const maxBufferSize = 4 << 10
const growthSizeFactor = 2 // higher value is faster
const growthRateFactor = 2 // higher value is slower
// By default, grow if below the maximum buffer size.
grow := cap(e.Buf) <= maxBufferSize/growthSizeFactor
// Growing can be expensive, so only grow
// if a sufficient number of bytes have been processed.
grow = grow && int64(cap(e.Buf)) < e.previousOffsetEnd()/growthRateFactor
if grow {
e.Buf = make([]byte, 0, cap(e.Buf)*growthSizeFactor)
}
return nil
}
// injectSyntacticErrorWithPosition wraps a SyntacticError with the position,
// otherwise it returns the error as is.
// It takes a position relative to the start of the start of e.buf.
func (e *encodeBuffer) injectSyntacticErrorWithPosition(err error, pos int) error {
if serr, ok := err.(*SyntacticError); ok {
return serr.withOffset(e.baseOffset + int64(pos))
}
return err
}
func (e *encodeBuffer) previousOffsetEnd() int64 { return e.baseOffset + int64(len(e.Buf)) }
func (e *encodeBuffer) unflushedBuffer() []byte { return e.Buf }
// avoidFlush indicates whether to avoid flushing to ensure there is always
// enough in the buffer to unwrite the last object member if it were empty.
func (e *encoderState) avoidFlush() bool {
switch {
case e.Tokens.Last.Length() == 0:
// Never flush after ObjectStart or ArrayStart since we don't know yet
// if the object or array will end up being empty.
return true
case e.Tokens.Last.needObjectValue():
// Never flush before the object value since we don't know yet
// if the object value will end up being empty.
return true
case e.Tokens.Last.NeedObjectName() && len(e.Buf) >= 2:
// Never flush after the object value if it does turn out to be empty.
switch string(e.Buf[len(e.Buf)-2:]) {
case `ll`, `""`, `{}`, `[]`: // last two bytes of every empty value
return true
}
}
return false
}
// UnwriteEmptyObjectMember unwrites the last object member if it is empty
// and reports whether it performed an unwrite operation.
func (e *encoderState) UnwriteEmptyObjectMember(prevName *string) bool {
if last := e.Tokens.Last; !last.isObject() || !last.NeedObjectName() || last.Length() == 0 {
panic("BUG: must be called on an object after writing a value")
}
// The flushing logic is modified to never flush a trailing empty value.
// The encoder never writes trailing whitespace eagerly.
b := e.unflushedBuffer()
// Detect whether the last value was empty.
var n int
if len(b) >= 3 {
switch string(b[len(b)-2:]) {
case "ll": // last two bytes of `null`
n = len(`null`)
case `""`:
// It is possible for a non-empty string to have `""` as a suffix
// if the second to the last quote was escaped.
if b[len(b)-3] == '\\' {
return false // e.g., `"\""` is not empty
}
n = len(`""`)
case `{}`:
n = len(`{}`)
case `[]`:
n = len(`[]`)
}
}
if n == 0 {
return false
}
// Unwrite the value, whitespace, colon, name, whitespace, and comma.
b = b[:len(b)-n]
b = jsonwire.TrimSuffixWhitespace(b)
b = jsonwire.TrimSuffixByte(b, ':')
b = jsonwire.TrimSuffixString(b)
b = jsonwire.TrimSuffixWhitespace(b)
b = jsonwire.TrimSuffixByte(b, ',')
e.Buf = b // store back truncated unflushed buffer
// Undo state changes.
e.Tokens.Last.decrement() // for object member value
e.Tokens.Last.decrement() // for object member name
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if e.Tokens.Last.isActiveNamespace() {
e.Namespaces.Last().removeLast()
}
e.Names.clearLast()
if prevName != nil {
e.Names.copyQuotedBuffer(e.Buf) // required by objectNameStack.replaceLastUnquotedName
e.Names.replaceLastUnquotedName(*prevName)
}
}
return true
}
// UnwriteOnlyObjectMemberName unwrites the only object member name
// and returns the unquoted name.
func (e *encoderState) UnwriteOnlyObjectMemberName() string {
if last := e.Tokens.Last; !last.isObject() || last.Length() != 1 {
panic("BUG: must be called on an object after writing first name")
}
// Unwrite the name and whitespace.
b := jsonwire.TrimSuffixString(e.Buf)
isVerbatim := bytes.IndexByte(e.Buf[len(b):], '\\') < 0
name := string(jsonwire.UnquoteMayCopy(e.Buf[len(b):], isVerbatim))
e.Buf = jsonwire.TrimSuffixWhitespace(b)
// Undo state changes.
e.Tokens.Last.decrement()
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
if e.Tokens.Last.isActiveNamespace() {
e.Namespaces.Last().removeLast()
}
e.Names.clearLast()
}
return name
}
// WriteToken writes the next token and advances the internal write offset.
//
// The provided token kind must be consistent with the JSON grammar.
// For example, it is an error to provide a number when the encoder
// is expecting an object name (which is always a string), or
// to provide an end object delimiter when the encoder is finishing an array.
// If the provided token is invalid, then it reports a [SyntacticError] and
// the internal state remains unchanged. The offset reported
// in [SyntacticError] will be relative to the [Encoder.OutputOffset].
func (e *Encoder) WriteToken(t Token) error {
return e.s.WriteToken(t)
}
func (e *encoderState) WriteToken(t Token) error {
k := t.Kind()
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.Expand) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the token
// Append the token to the output and to the state machine.
var err error
switch k {
case 'n':
b = append(b, "null"...)
err = e.Tokens.appendLiteral()
case 'f':
b = append(b, "false"...)
err = e.Tokens.appendLiteral()
case 't':
b = append(b, "true"...)
err = e.Tokens.appendLiteral()
case '"':
if b, err = t.appendString(b, &e.Flags); err != nil {
break
}
if !e.Flags.Get(jsonflags.AllowDuplicateNames) && e.Tokens.Last.NeedObjectName() {
if !e.Tokens.Last.isValidNamespace() {
err = errInvalidNamespace
break
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
err = newDuplicateNameError(b[pos:])
break
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
err = e.Tokens.appendString()
case '0':
if b, err = t.appendNumber(b, e.Flags.Get(jsonflags.CanonicalizeNumbers)); err != nil {
break
}
err = e.Tokens.appendNumber()
case '{':
b = append(b, '{')
if err = e.Tokens.pushObject(); err != nil {
break
}
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Names.push()
e.Namespaces.push()
}
case '}':
b = append(b, '}')
if err = e.Tokens.popObject(); err != nil {
break
}
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Names.pop()
e.Namespaces.pop()
}
case '[':
b = append(b, '[')
err = e.Tokens.pushArray()
case ']':
b = append(b, ']')
err = e.Tokens.popArray()
default:
err = &SyntacticError{str: "invalid json.Token"}
}
if err != nil {
return e.injectSyntacticErrorWithPosition(err, pos)
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// AppendRaw appends either a raw string (without double quotes) or number.
// Specify safeASCII if the string output is guaranteed to be ASCII
// without any characters (including '<', '>', and '&') that need escaping,
// otherwise this will validate whether the string needs escaping.
// The appended bytes for a JSON number must be valid.
//
// This is a specialized implementation of Encoder.WriteValue
// that allows appending directly into the buffer.
// It is only called from marshal logic in the "json" package.
func (e *encoderState) AppendRaw(k Kind, safeASCII bool, appendFn func([]byte) ([]byte, error)) error {
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.Expand) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the token
var err error
switch k {
case '"':
// Append directly into the encoder buffer by assuming that
// most of the time none of the characters need escaping.
b = append(b, '"')
if b, err = appendFn(b); err != nil {
return err
}
b = append(b, '"')
// Check whether we need to escape the string and if necessary
// copy it to a scratch buffer and then escape it back.
isVerbatim := safeASCII || !jsonwire.NeedEscape(b[pos+len(`"`):len(b)-len(`"`)])
if !isVerbatim {
var err error
b2 := append(e.unusedCache, b[pos+len(`"`):len(b)-len(`"`)]...)
b, err = jsonwire.AppendQuote(b[:pos], string(b2), &e.Flags)
e.unusedCache = b2[:0]
if err != nil {
return e.injectSyntacticErrorWithPosition(err, pos)
}
}
// Update the state machine.
if !e.Flags.Get(jsonflags.AllowDuplicateNames) && e.Tokens.Last.NeedObjectName() {
if !e.Tokens.Last.isValidNamespace() {
return errInvalidNamespace
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], isVerbatim) {
err := newDuplicateNameError(b[pos:])
return e.injectSyntacticErrorWithPosition(err, pos)
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
if err := e.Tokens.appendString(); err != nil {
return e.injectSyntacticErrorWithPosition(err, pos)
}
case '0':
if b, err = appendFn(b); err != nil {
return err
}
if err := e.Tokens.appendNumber(); err != nil {
return e.injectSyntacticErrorWithPosition(err, pos)
}
default:
panic("BUG: invalid kind")
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// WriteValue writes the next raw value and advances the internal write offset.
// The Encoder does not simply copy the provided value verbatim, but
// parses it to ensure that it is syntactically valid and reformats it
// according to how the Encoder is configured to format whitespace and strings.
// If [AllowInvalidUTF8] is specified, then any invalid UTF-8 is mangled
// as the Unicode replacement character, U+FFFD.
//
// The provided value kind must be consistent with the JSON grammar
// (see examples on [Encoder.WriteToken]). If the provided value is invalid,
// then it reports a [SyntacticError] and the internal state remains unchanged.
// The offset reported in [SyntacticError] will be relative to the
// [Encoder.OutputOffset] plus the offset into v of any encountered syntax error.
func (e *Encoder) WriteValue(v Value) error {
return e.s.WriteValue(v)
}
func (e *encoderState) WriteValue(v Value) error {
e.maxValue |= len(v) // bitwise OR is a fast approximation of max
k := v.Kind()
b := e.Buf // use local variable to avoid mutating e in case of error
// Append any delimiters or optional whitespace.
b = e.Tokens.MayAppendDelim(b, k)
if e.Flags.Get(jsonflags.Expand) {
b = e.appendWhitespace(b, k)
}
pos := len(b) // offset before the value
// Append the value the output.
var n int
n += jsonwire.ConsumeWhitespace(v[n:])
b, m, err := e.reformatValue(b, v[n:], e.Tokens.Depth())
if err != nil {
return e.injectSyntacticErrorWithPosition(err, pos+n+m)
}
n += m
n += jsonwire.ConsumeWhitespace(v[n:])
if len(v) > n {
err = newInvalidCharacterError(v[n:], "after top-level value")
return e.injectSyntacticErrorWithPosition(err, pos+n)
}
// Append the kind to the state machine.
switch k {
case 'n', 'f', 't':
err = e.Tokens.appendLiteral()
case '"':
if !e.Flags.Get(jsonflags.AllowDuplicateNames) && e.Tokens.Last.NeedObjectName() {
if !e.Tokens.Last.isValidNamespace() {
err = errInvalidNamespace
break
}
if e.Tokens.Last.isActiveNamespace() && !e.Namespaces.Last().insertQuoted(b[pos:], false) {
err = newDuplicateNameError(b[pos:])
break
}
e.Names.ReplaceLastQuotedOffset(pos) // only replace if insertQuoted succeeds
}
err = e.Tokens.appendString()
case '0':
err = e.Tokens.appendNumber()
case '{':
if err = e.Tokens.pushObject(); err != nil {
break
}
if err = e.Tokens.popObject(); err != nil {
panic("BUG: popObject should never fail immediately after pushObject: " + err.Error())
}
case '[':
if err = e.Tokens.pushArray(); err != nil {
break
}
if err = e.Tokens.popArray(); err != nil {
panic("BUG: popArray should never fail immediately after pushArray: " + err.Error())
}
}
if err != nil {
return e.injectSyntacticErrorWithPosition(err, pos)
}
// Finish off the buffer and store it back into e.
e.Buf = b
if e.NeedFlush() {
return e.Flush()
}
return nil
}
// appendWhitespace appends whitespace that immediately precedes the next token.
func (e *encoderState) appendWhitespace(b []byte, next Kind) []byte {
if e.Tokens.needDelim(next) == ':' {
return append(b, ' ')
} else {
return e.AppendIndent(b, e.Tokens.NeedIndent(next))
}
}
// AppendIndent appends the appropriate number of indentation characters
// for the current nested level, n.
func (e *encoderState) AppendIndent(b []byte, n int) []byte {
if n == 0 {
return b
}
b = append(b, '\n')
b = append(b, e.IndentPrefix...)
for ; n > 1; n-- {
b = append(b, e.Indent...)
}
return b
}
// reformatValue parses a JSON value from the start of src and
// appends it to the end of dst, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatValue(dst []byte, src Value, depth int) ([]byte, int, error) {
// TODO: Should this update ValueFlags as input?
if len(src) == 0 {
return dst, 0, io.ErrUnexpectedEOF
}
switch k := Kind(src[0]).normalize(); k {
case 'n':
if jsonwire.ConsumeNull(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "null")
return dst, n, err
}
return append(dst, "null"...), len("null"), nil
case 'f':
if jsonwire.ConsumeFalse(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "false")
return dst, n, err
}
return append(dst, "false"...), len("false"), nil
case 't':
if jsonwire.ConsumeTrue(src) == 0 {
n, err := jsonwire.ConsumeLiteral(src, "true")
return dst, n, err
}
return append(dst, "true"...), len("true"), nil
case '"':
if n := jsonwire.ConsumeSimpleString(src); n > 0 {
dst, src = append(dst, src[:n]...), src[n:] // copy simple strings verbatim
return dst, n, nil
}
return jsonwire.ReformatString(dst, src, &e.Flags)
case '0':
if n := jsonwire.ConsumeSimpleNumber(src); n > 0 && !e.Flags.Get(jsonflags.CanonicalizeNumbers) {
dst, src = append(dst, src[:n]...), src[n:] // copy simple numbers verbatim
return dst, n, nil
}
return jsonwire.ReformatNumber(dst, src, e.Flags.Get(jsonflags.CanonicalizeNumbers))
case '{':
return e.reformatObject(dst, src, depth)
case '[':
return e.reformatArray(dst, src, depth)
default:
return dst, 0, newInvalidCharacterError(src, "at start of value")
}
}
// reformatObject parses a JSON object from the start of src and
// appends it to the end of src, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatObject(dst []byte, src Value, depth int) ([]byte, int, error) {
// Append object start.
if len(src) == 0 || src[0] != '{' {
panic("BUG: reformatObject must be called with a buffer that starts with '{'")
} else if depth == maxNestingDepth+1 {
return dst, 0, errMaxDepth
}
dst = append(dst, '{')
n := len("{")
// Append (possible) object end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
if src[n] == '}' {
dst = append(dst, '}')
n += len("}")
return dst, n, nil
}
var err error
var names *objectNamespace
if !e.Flags.Get(jsonflags.AllowDuplicateNames) {
e.Namespaces.push()
defer e.Namespaces.pop()
names = e.Namespaces.Last()
}
depth++
for {
// Append optional newline and indentation.
if e.Flags.Get(jsonflags.Expand) {
dst = e.AppendIndent(dst, depth)
}
// Append object name.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
m := jsonwire.ConsumeSimpleString(src[n:])
if m > 0 {
dst = append(dst, src[n:n+m]...)
} else {
dst, m, err = jsonwire.ReformatString(dst, src[n:], &e.Flags)
if err != nil {
return dst, n + m, err
}
}
// TODO: Specify whether the name is verbatim or not.
if !e.Flags.Get(jsonflags.AllowDuplicateNames) && !names.insertQuoted(src[n:n+m], false) {
return dst, n, newDuplicateNameError(src[n : n+m])
}
n += m
// Append colon.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
if src[n] != ':' {
return dst, n, newInvalidCharacterError(src[n:], "after object name (expecting ':')")
}
dst = append(dst, ':')
n += len(":")
if e.Flags.Get(jsonflags.Expand) {
dst = append(dst, ' ')
}
// Append object value.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
dst, m, err = e.reformatValue(dst, src[n:], depth)
if err != nil {
return dst, n + m, err
}
n += m
// Append comma or object end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
switch src[n] {
case ',':
dst = append(dst, ',')
n += len(",")
continue
case '}':
if e.Flags.Get(jsonflags.Expand) {
dst = e.AppendIndent(dst, depth-1)
}
dst = append(dst, '}')
n += len("}")
return dst, n, nil
default:
return dst, n, newInvalidCharacterError(src[n:], "after object value (expecting ',' or '}')")
}
}
}
// reformatArray parses a JSON array from the start of src and
// appends it to the end of dst, reformatting whitespace and strings as needed.
// It returns the extended dst buffer and the number of consumed input bytes.
func (e *encoderState) reformatArray(dst []byte, src Value, depth int) ([]byte, int, error) {
// Append array start.
if len(src) == 0 || src[0] != '[' {
panic("BUG: reformatArray must be called with a buffer that starts with '['")
} else if depth == maxNestingDepth+1 {
return dst, 0, errMaxDepth
}
dst = append(dst, '[')
n := len("[")
// Append (possible) array end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
if src[n] == ']' {
dst = append(dst, ']')
n += len("]")
return dst, n, nil
}
var err error
depth++
for {
// Append optional newline and indentation.
if e.Flags.Get(jsonflags.Expand) {
dst = e.AppendIndent(dst, depth)
}
// Append array value.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
var m int
dst, m, err = e.reformatValue(dst, src[n:], depth)
if err != nil {
return dst, n + m, err
}
n += m
// Append comma or array end.
n += jsonwire.ConsumeWhitespace(src[n:])
if uint(len(src)) <= uint(n) {
return dst, n, io.ErrUnexpectedEOF
}
switch src[n] {
case ',':
dst = append(dst, ',')
n += len(",")
continue
case ']':
if e.Flags.Get(jsonflags.Expand) {
dst = e.AppendIndent(dst, depth-1)
}
dst = append(dst, ']')
n += len("]")
return dst, n, nil
default:
return dst, n, newInvalidCharacterError(src[n:], "after array value (expecting ',' or ']')")
}
}
}
// OutputOffset returns the current output byte offset. It gives the location
// of the next byte immediately after the most recently written token or value.
// The number of bytes actually written to the underlying [io.Writer] may be less
// than this offset due to internal buffering effects.
func (e *Encoder) OutputOffset() int64 {
return e.s.previousOffsetEnd()
}
// UnusedBuffer returns a zero-length buffer with a possible non-zero capacity.
// This buffer is intended to be used to populate a [Value]
// being passed to an immediately succeeding [Encoder.WriteValue] call.
//
// Example usage:
//
// b := d.UnusedBuffer()
// b = append(b, '"')
// b = appendString(b, v) // append the string formatting of v
// b = append(b, '"')
// ... := d.WriteValue(b)
//
// It is the user's responsibility to ensure that the value is valid JSON.
func (e *Encoder) UnusedBuffer() []byte {
// NOTE: We don't return e.buf[len(e.buf):cap(e.buf)] since WriteValue would
// need to take special care to avoid mangling the data while reformatting.
// WriteValue can't easily identify whether the input Value aliases e.buf
// without using unsafe.Pointer. Thus, we just return a different buffer.
// Should this ever alias e.buf, we need to consider how it operates with
// the specialized performance optimization for bytes.Buffer.
n := 1 << bits.Len(uint(e.s.maxValue|63)) // fast approximation for max length
if cap(e.s.unusedCache) < n {
e.s.unusedCache = make([]byte, 0, n)
}
return e.s.unusedCache
}
// StackDepth returns the depth of the state machine for written JSON data.
// Each level on the stack represents a nested JSON object or array.
// It is incremented whenever an [ObjectStart] or [ArrayStart] token is encountered
// and decremented whenever an [ObjectEnd] or [ArrayEnd] token is encountered.
// The depth is zero-indexed, where zero represents the top-level JSON value.
func (e *Encoder) StackDepth() int {
// NOTE: Keep in sync with Decoder.StackDepth.
return e.s.Tokens.Depth() - 1
}
// StackIndex returns information about the specified stack level.
// It must be a number between 0 and [Encoder.StackDepth], inclusive.
// For each level, it reports the kind:
//
// - 0 for a level of zero,
// - '{' for a level representing a JSON object, and
// - '[' for a level representing a JSON array.
//
// It also reports the length of that JSON object or array.
// Each name and value in a JSON object is counted separately,
// so the effective number of members would be half the length.
// A complete JSON object must have an even length.
func (e *Encoder) StackIndex(i int) (Kind, int) {
// NOTE: Keep in sync with Decoder.StackIndex.
switch s := e.s.Tokens.index(i); {
case i > 0 && s.isObject():
return '{', s.Length()
case i > 0 && s.isArray():
return '[', s.Length()
default:
return 0, s.Length()
}
}
// StackPointer returns a JSON Pointer (RFC 6901) to the most recently written value.
// Object names are only present if [AllowDuplicateNames] is false, otherwise
// object members are represented using their index within the object.
func (e *Encoder) StackPointer() string {
e.s.Names.copyQuotedBuffer(e.s.Buf)
return string(e.s.appendStackPointer(nil))
}
+60
View File
@@ -0,0 +1,60 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"github.com/go-json-experiment/json/internal/jsonwire"
)
const errorPrefix = "jsontext: "
type ioError struct {
action string // either "read" or "write"
err error
}
func (e *ioError) Error() string {
return errorPrefix + e.action + " error: " + e.err.Error()
}
func (e *ioError) Unwrap() error {
return e.err
}
// SyntacticError is a description of a syntactic error that occurred when
// encoding or decoding JSON according to the grammar.
//
// The contents of this error as produced by this package may change over time.
type SyntacticError struct {
requireKeyedLiterals
nonComparable
// ByteOffset indicates that an error occurred after this byte offset.
ByteOffset int64
str string
}
func (e *SyntacticError) Error() string {
return errorPrefix + e.str
}
func (e *SyntacticError) withOffset(pos int64) error {
return &SyntacticError{ByteOffset: pos, str: e.str}
}
func newDuplicateNameError[Bytes ~[]byte | ~string](quoted Bytes) *SyntacticError {
return &SyntacticError{str: "duplicate name " + string(quoted) + " in object"}
}
func newInvalidCharacterError[Bytes ~[]byte | ~string](prefix Bytes, where string) *SyntacticError {
what := jsonwire.QuoteRune(prefix)
return &SyntacticError{str: "invalid character " + what + " " + where}
}
// TODO: Error types between "json", "jsontext", and "jsonwire" is a mess.
// Clean this up.
func init() {
// Inject behavior in "jsonwire" so that it can produce SyntacticError types.
jsonwire.NewError = func(s string) error { return &SyntacticError{str: s} }
jsonwire.ErrInvalidUTF8 = &SyntacticError{str: jsonwire.ErrInvalidUTF8.Error()}
}
+83
View File
@@ -0,0 +1,83 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"io"
"github.com/go-json-experiment/json/internal"
)
// Internal is for internal use only.
// This is exempt from the Go compatibility agreement.
var Internal exporter
type exporter struct{}
// Export exposes internal functionality from "jsontext" to "json".
// This cannot be dynamically called by other packages since
// they cannot obtain a reference to the internal.AllowInternalUse value.
func (exporter) Export(p *internal.NotForPublicUse) export {
if p != &internal.AllowInternalUse {
panic("unauthorized call to Export")
}
return export{}
}
// The export type exposes functionality to packages with visibility to
// the internal.AllowInternalUse variable. The "json" package uses this
// to modify low-level state in the Encoder and Decoder types.
// It mutates the state directly instead of calling ReadToken or WriteToken
// since this is more performant. The public APIs need to track state to ensure
// that users are constructing a valid JSON value, but the "json" implementation
// guarantees that it emits valid JSON by the structure of the code itself.
type export struct{}
// Encoder returns a pointer to the underlying encoderState.
func (export) Encoder(e *Encoder) *encoderState { return &e.s }
// Decoder returns a pointer to the underlying decoderState.
func (export) Decoder(d *Decoder) *decoderState { return &d.s }
func (export) GetBufferedEncoder(o ...Options) *Encoder {
return getBufferedEncoder(o...)
}
func (export) PutBufferedEncoder(e *Encoder) {
putBufferedEncoder(e)
}
func (export) GetStreamingEncoder(w io.Writer, o ...Options) *Encoder {
return getStreamingEncoder(w, o...)
}
func (export) PutStreamingEncoder(e *Encoder) {
putStreamingEncoder(e)
}
func (export) GetBufferedDecoder(b []byte, o ...Options) *Decoder {
return getBufferedDecoder(b, o...)
}
func (export) PutBufferedDecoder(d *Decoder) {
putBufferedDecoder(d)
}
func (export) GetStreamingDecoder(r io.Reader, o ...Options) *Decoder {
return getStreamingDecoder(r, o...)
}
func (export) PutStreamingDecoder(d *Decoder) {
putStreamingDecoder(d)
}
func (export) NewDuplicateNameError(quoted []byte, pos int64) error {
return newDuplicateNameError(quoted).withOffset(pos)
}
func (export) NewInvalidCharacterError(prefix, where string, pos int64) error {
return newInvalidCharacterError(prefix, where).withOffset(pos)
}
func (export) NewMissingNameError(pos int64) error {
return errMissingName.withOffset(pos)
}
func (export) NewInvalidUTF8Error(pos int64) error {
return errInvalidUTF8.withOffset(pos)
}
+178
View File
@@ -0,0 +1,178 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"strings"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonopts"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// Options configures [NewEncoder], [Encoder.Reset], [NewDecoder],
// and [Decoder.Reset] with specific features.
// Each function takes in a variadic list of options, where properties
// set in latter options override the value of previously set properties.
//
// The Options type is identical to [encoding/json.Options] and
// [encoding/json/v2.Options]. Options from the other packages may
// be passed to functionality in this package, but are ignored.
// Options from this package may be used with the other packages.
type Options = jsonopts.Options
// AllowDuplicateNames specifies that JSON objects may contain
// duplicate member names. Disabling the duplicate name check may provide
// performance benefits, but breaks compliance with RFC 7493, section 2.3.
// The input or output will still be compliant with RFC 8259,
// which leaves the handling of duplicate names as unspecified behavior.
//
// This affects either encoding or decoding.
func AllowDuplicateNames(v bool) Options {
if v {
return jsonflags.AllowDuplicateNames | 1
} else {
return jsonflags.AllowDuplicateNames | 0
}
}
// AllowInvalidUTF8 specifies that JSON strings may contain invalid UTF-8,
// which will be mangled as the Unicode replacement character, U+FFFD.
// This causes the encoder or decoder to break compliance with
// RFC 7493, section 2.1, and RFC 8259, section 8.1.
//
// This affects either encoding or decoding.
func AllowInvalidUTF8(v bool) Options {
if v {
return jsonflags.AllowInvalidUTF8 | 1
} else {
return jsonflags.AllowInvalidUTF8 | 0
}
}
// EscapeForHTML specifies that '<', '>', and '&' characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u003c) so that
// the output is safe to embed within HTML.
//
// This only affects encoding and is ignored when decoding.
func EscapeForHTML(v bool) Options {
if v {
return jsonflags.EscapeForHTML | 1
} else {
return jsonflags.EscapeForHTML | 0
}
}
// EscapeForJS specifies that U+2028 and U+2029 characters within JSON strings
// should be escaped as a hexadecimal Unicode codepoint (e.g., \u2028) so that
// the output is valid to embed within JavaScript. See RFC 8259, section 12.
//
// This only affects encoding and is ignored when decoding.
func EscapeForJS(v bool) Options {
if v {
return jsonflags.EscapeForJS | 1
} else {
return jsonflags.EscapeForJS | 0
}
}
// Expand specifies that the JSON output should be expanded,
// where every JSON object member or JSON array element
// appears on a new, indented line according to the nesting depth.
// If an indent is not already specified, then it defaults to using "\t".
//
// If set to false, then the output is compact,
// where no whitespace is emitted between JSON values.
//
// This only affects encoding and is ignored when decoding.
func Expand(v bool) Options {
if v {
return jsonflags.Expand | 1
} else {
return jsonflags.Expand | 0
}
}
// WithIndent specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix (see [WithIndentPrefix])
// followed by one or more copies of indent according to the nesting depth.
// The indent must only be composed of space or tab characters.
//
// If the intent to emit indented output without a preference for
// the particular indent string, then use [Expand] instead.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Expand] being set to true.
func WithIndent(indent string) Options {
// Fast-path: Return a constant for common indents, which avoids allocating.
// These are derived from analyzing the Go module proxy on 2023-07-01.
switch indent {
case "\t":
return jsonopts.Indent("\t") // ~14k usages
case " ":
return jsonopts.Indent(" ") // ~18k usages
case " ":
return jsonopts.Indent(" ") // ~1.7k usages
case " ":
return jsonopts.Indent(" ") // ~52k usages
case " ":
return jsonopts.Indent(" ") // ~12k usages
case "":
return jsonopts.Indent("") // ~1.5k usages
}
// Otherwise, allocate for this unique value.
if s := strings.Trim(indent, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent")
}
return jsonopts.Indent(indent)
}
// WithIndentPrefix specifies that the encoder should emit multiline output
// where each element in a JSON object or array begins on a new, indented line
// beginning with the indent prefix followed by one or more copies of indent
// (see [WithIndent]) according to the nesting depth.
// The prefix must only be composed of space or tab characters.
//
// This only affects encoding and is ignored when decoding.
// Use of this option implies [Expand] being set to true.
func WithIndentPrefix(prefix string) Options {
if s := strings.Trim(prefix, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent prefix")
}
return jsonopts.IndentPrefix(prefix)
}
/*
// TODO(https://go.dev/issue/56733): Implement WithByteLimit and WithDepthLimit.
// WithByteLimit sets a limit on the number of bytes of input or output bytes
// that may be consumed or produced for each top-level JSON value.
// If a [Decoder] or [Encoder] method call would need to consume/produce
// more than a total of n bytes to make progress on the top-level JSON value,
// then the call will report an error.
// Whitespace before and within the top-level value are counted against the limit.
// Whitespace after a top-level value are counted against the limit
// for the next top-level value.
//
// A non-positive limit is equivalent to no limit at all.
// If unspecified, the default limit is no limit at all.
func WithByteLimit(n int64) Options {
return jsonopts.ByteLimit(max(n, 0))
}
// WithDepthLimit sets a limit on the maximum depth of JSON nesting
// that may be consumed or produced for each top-level JSON value.
// If a [Decoder] or [Encoder] method call would need to consume or produce
// a depth greater than n to make progress on the top-level JSON value,
// then the call will report an error.
//
// A non-positive limit is equivalent to no limit at all.
// If unspecified, the default limit is 10000.
func WithDepthLimit(n int) Options {
return jsonopts.DepthLimit(max(n, 0))
}
*/
+150
View File
@@ -0,0 +1,150 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"bytes"
"io"
"math/bits"
"sync"
)
// TODO(https://go.dev/issue/47657): Use sync.PoolOf.
var (
// This owns the internal buffer since there is no io.Writer to output to.
// Since the buffer can get arbitrarily large in normal usage,
// there is statistical tracking logic to determine whether to recycle
// the internal buffer or not based on a history of utilization.
bufferedEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON before flushing it to the underlying io.Writer.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferEncoderPool = &sync.Pool{New: func() any { return new(Encoder) }}
)
// bufferStatistics is statistics to track buffer utilization.
// It is used to determine whether to recycle a buffer or not
// to avoid https://go.dev/issue/23199.
type bufferStatistics struct {
strikes int // number of times the buffer was under-utilized
prevLen int // length of previous buffer
}
func getBufferedEncoder(opts ...Options) *Encoder {
e := bufferedEncoderPool.Get().(*Encoder)
if e.s.Buf == nil {
// Round up to nearest 2ⁿ to make best use of malloc size classes.
// See runtime/sizeclasses.go on Go1.15.
// Logical OR with 63 to ensure 64 as the minimum buffer size.
n := 1 << bits.Len(uint(e.s.bufStats.prevLen|63))
e.s.Buf = make([]byte, 0, n)
}
e.s.reset(e.s.Buf[:0], nil, opts...)
return e
}
func putBufferedEncoder(e *Encoder) {
// Recycle large buffers only if sufficiently utilized.
// If a buffer is under-utilized enough times sequentially,
// then it is discarded, ensuring that a single large buffer
// won't be kept alive by a continuous stream of small usages.
//
// The worst case utilization is computed as:
// MIN_UTILIZATION_THRESHOLD / (1 + MAX_NUM_STRIKES)
//
// For the constants chosen below, this is (25%)/(1+4) ⇒ 5%.
// This may seem low, but it ensures a lower bound on
// the absolute worst-case utilization. Without this check,
// this would be theoretically 0%, which is infinitely worse.
//
// See https://go.dev/issue/27735.
switch {
case cap(e.s.Buf) <= 4<<10: // always recycle buffers smaller than 4KiB
e.s.bufStats.strikes = 0
case cap(e.s.Buf)/4 <= len(e.s.Buf): // at least 25% utilization
e.s.bufStats.strikes = 0
case e.s.bufStats.strikes < 4: // at most 4 strikes
e.s.bufStats.strikes++
default: // discard the buffer; too large and too often under-utilized
e.s.bufStats.strikes = 0
e.s.bufStats.prevLen = len(e.s.Buf) // heuristic for size to allocate next time
e.s.Buf = nil
}
bufferedEncoderPool.Put(e)
}
func getStreamingEncoder(w io.Writer, opts ...Options) *Encoder {
if _, ok := w.(*bytes.Buffer); ok {
e := bytesBufferEncoderPool.Get().(*Encoder)
e.s.reset(nil, w, opts...) // buffer taken from bytes.Buffer
return e
} else {
e := streamingEncoderPool.Get().(*Encoder)
e.s.reset(e.s.Buf[:0], w, opts...) // preserve existing buffer
return e
}
}
func putStreamingEncoder(e *Encoder) {
if _, ok := e.s.wr.(*bytes.Buffer); ok {
bytesBufferEncoderPool.Put(e)
} else {
if cap(e.s.Buf) > 64<<10 {
e.s.Buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingEncoderPool.Put(e)
}
}
var (
// This does not own the internal buffer since it is externally provided.
bufferedDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This owns the internal buffer, but it is only used to temporarily store
// buffered JSON fetched from the underlying io.Reader.
// In a sufficiently efficient streaming mode, we do not expect the buffer
// to grow arbitrarily large. Thus, we avoid recycling large buffers.
streamingDecoderPool = &sync.Pool{New: func() any { return new(Decoder) }}
// This does not own the internal buffer since
// it is taken directly from the provided bytes.Buffer.
bytesBufferDecoderPool = bufferedDecoderPool
)
func getBufferedDecoder(b []byte, opts ...Options) *Decoder {
d := bufferedDecoderPool.Get().(*Decoder)
d.s.reset(b, nil, opts...)
return d
}
func putBufferedDecoder(d *Decoder) {
bufferedDecoderPool.Put(d)
}
func getStreamingDecoder(r io.Reader, opts ...Options) *Decoder {
if _, ok := r.(*bytes.Buffer); ok {
d := bytesBufferDecoderPool.Get().(*Decoder)
d.s.reset(nil, r, opts...) // buffer taken from bytes.Buffer
return d
} else {
d := streamingDecoderPool.Get().(*Decoder)
d.s.reset(d.s.buf[:0], r, opts...) // preserve existing buffer
return d
}
}
func putStreamingDecoder(d *Decoder) {
if _, ok := d.s.rd.(*bytes.Buffer); ok {
bytesBufferDecoderPool.Put(d)
} else {
if cap(d.s.buf) > 64<<10 {
d.s.buf = nil // avoid pinning arbitrarily large amounts of memory
}
streamingDecoderPool.Put(d)
}
}
+31
View File
@@ -0,0 +1,31 @@
// Copyright 2023 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
var errInvalidUTF8 = &SyntacticError{str: "invalid UTF-8 within string"}
// AppendQuote appends a double-quoted JSON string literal representing src
// to dst and returns the extended buffer.
// It uses the minimal string representation per RFC 8785, section 3.2.2.2.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
func AppendQuote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
return jsonwire.AppendQuote(dst, src, &jsonflags.Flags{})
}
// AppendUnquote appends the decoded interpretation of src as a
// double-quoted JSON string literal to dst and returns the extended buffer.
// The input src must be a JSON string without any surrounding whitespace.
// Invalid UTF-8 bytes are replaced with the Unicode replacement character
// and an error is returned at the end indicating the presence of invalid UTF-8.
// Any trailing bytes after the JSON string literal results in an error.
func AppendUnquote[Bytes ~[]byte | ~string](dst []byte, src Bytes) ([]byte, error) {
return jsonwire.AppendUnquote(dst, src)
}
+715
View File
@@ -0,0 +1,715 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"math"
"strconv"
"github.com/go-json-experiment/json/internal/jsonwire"
)
var (
errMissingName = &SyntacticError{str: "missing string for object name"}
errMissingColon = &SyntacticError{str: "missing character ':' after object name"}
errMissingValue = &SyntacticError{str: "missing value after object name"}
errMissingComma = &SyntacticError{str: "missing character ',' after object or array value"}
errMismatchDelim = &SyntacticError{str: "mismatching structural token for object or array"}
errMaxDepth = &SyntacticError{str: "exceeded max depth"}
errInvalidNamespace = &SyntacticError{str: "object namespace is in an invalid state"}
)
// Per RFC 8259, section 9, implementations may enforce a maximum depth.
// Such a limit is necessary to prevent stack overflows.
const maxNestingDepth = 10000
type state struct {
// Tokens validates whether the next token kind is valid.
Tokens stateMachine
// Names is a stack of object names.
// Not used if AllowDuplicateNames is true.
Names objectNameStack
// Namespaces is a stack of object namespaces.
// For performance reasons, Encoder or Decoder may not update this
// if Marshal or Unmarshal is able to track names in a more efficient way.
// See makeMapArshaler and makeStructArshaler.
// Not used if AllowDuplicateNames is true.
Namespaces objectNamespaceStack
}
func (s *state) reset() {
s.Tokens.reset()
s.Names.reset()
s.Namespaces.reset()
}
// appendStackPointer appends a JSON Pointer (RFC 6901) to the current value.
// The returned pointer is only accurate if s.names is populated,
// otherwise it uses the numeric index as the object member name.
//
// Invariant: Must call s.names.copyQuotedBuffer beforehand.
func (s state) appendStackPointer(b []byte) []byte {
var objectDepth int
for i := 1; i < s.Tokens.Depth(); i++ {
e := s.Tokens.index(i)
if e.Length() == 0 {
break // empty object or array
}
b = append(b, '/')
switch {
case e.isObject():
if objectDepth < s.Names.length() {
for _, c := range s.Names.getUnquoted(objectDepth) {
// Per RFC 6901, section 3, escape '~' and '/' characters.
switch c {
case '~':
b = append(b, "~0"...)
case '/':
b = append(b, "~1"...)
default:
b = append(b, c)
}
}
} else {
// Since the names stack is unpopulated, the name is unknown.
// As a best-effort replacement, use the numeric member index.
// While inaccurate, it produces a syntactically valid pointer.
b = strconv.AppendUint(b, uint64((e.Length()-1)/2), 10)
}
objectDepth++
case e.isArray():
b = strconv.AppendUint(b, uint64(e.Length()-1), 10)
}
}
return b
}
// stateMachine is a push-down automaton that validates whether
// a sequence of tokens is valid or not according to the JSON grammar.
// It is useful for both encoding and decoding.
//
// It is a stack where each entry represents a nested JSON object or array.
// The stack has a minimum depth of 1 where the first level is a
// virtual JSON array to handle a stream of top-level JSON values.
// The top-level virtual JSON array is special in that it doesn't require commas
// between each JSON value.
//
// For performance, most methods are carefully written to be inlinable.
// The zero value is a valid state machine ready for use.
type stateMachine struct {
Stack []stateEntry
Last stateEntry
}
// reset resets the state machine.
// The machine always starts with a minimum depth of 1.
func (m *stateMachine) reset() {
m.Stack = m.Stack[:0]
if cap(m.Stack) > 1<<10 {
m.Stack = nil
}
m.Last = stateTypeArray
}
// Depth is the current nested depth of JSON objects and arrays.
// It is one-indexed (i.e., top-level values have a depth of 1).
func (m stateMachine) Depth() int {
return len(m.Stack) + 1
}
// index returns a reference to the ith entry.
// It is only valid until the next push method call.
func (m *stateMachine) index(i int) *stateEntry {
if i == len(m.Stack) {
return &m.Last
}
return &m.Stack[i]
}
// DepthLength reports the current nested depth and
// the length of the last JSON object or array.
func (m stateMachine) DepthLength() (int, int) {
return m.Depth(), m.Last.Length()
}
// appendLiteral appends a JSON literal as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendLiteral() error {
switch {
case m.Last.NeedObjectName():
return errMissingName
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last.Increment()
return nil
}
}
// appendString appends a JSON string as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendString() error {
switch {
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last.Increment()
return nil
}
}
// appendNumber appends a JSON number as the next token in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) appendNumber() error {
return m.appendLiteral()
}
// pushObject appends a JSON start object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushObject() error {
switch {
case m.Last.NeedObjectName():
return errMissingName
case !m.Last.isValidNamespace():
return errInvalidNamespace
case len(m.Stack) == maxNestingDepth:
return errMaxDepth
default:
m.Last.Increment()
m.Stack = append(m.Stack, m.Last)
m.Last = stateTypeObject
return nil
}
}
// popObject appends a JSON end object token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popObject() error {
switch {
case !m.Last.isObject():
return errMismatchDelim
case m.Last.needObjectValue():
return errMissingValue
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last = m.Stack[len(m.Stack)-1]
m.Stack = m.Stack[:len(m.Stack)-1]
return nil
}
}
// pushArray appends a JSON start array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) pushArray() error {
switch {
case m.Last.NeedObjectName():
return errMissingName
case !m.Last.isValidNamespace():
return errInvalidNamespace
case len(m.Stack) == maxNestingDepth:
return errMaxDepth
default:
m.Last.Increment()
m.Stack = append(m.Stack, m.Last)
m.Last = stateTypeArray
return nil
}
}
// popArray appends a JSON end array token as next in the sequence.
// If an error is returned, the state is not mutated.
func (m *stateMachine) popArray() error {
switch {
case !m.Last.isArray() || len(m.Stack) == 0: // forbid popping top-level virtual JSON array
return errMismatchDelim
case !m.Last.isValidNamespace():
return errInvalidNamespace
default:
m.Last = m.Stack[len(m.Stack)-1]
m.Stack = m.Stack[:len(m.Stack)-1]
return nil
}
}
// NeedIndent reports whether indent whitespace should be injected.
// A zero value means that no whitespace should be injected.
// A positive value means '\n', indentPrefix, and (n-1) copies of indentBody
// should be appended to the output immediately before the next token.
func (m stateMachine) NeedIndent(next Kind) (n int) {
willEnd := next == '}' || next == ']'
switch {
case m.Depth() == 1:
return 0 // top-level values are never indented
case m.Last.Length() == 0 && willEnd:
return 0 // an empty object or array is never indented
case m.Last.Length() == 0 || m.Last.needImplicitComma(next):
return m.Depth()
case willEnd:
return m.Depth() - 1
default:
return 0
}
}
// MayAppendDelim appends a colon or comma that may precede the next token.
func (m stateMachine) MayAppendDelim(b []byte, next Kind) []byte {
switch {
case m.Last.needImplicitColon():
return append(b, ':')
case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values
return append(b, ',')
default:
return b
}
}
// needDelim reports whether a colon or comma token should be implicitly emitted
// before the next token of the specified kind.
// A zero value means no delimiter should be emitted.
func (m stateMachine) needDelim(next Kind) (delim byte) {
switch {
case m.Last.needImplicitColon():
return ':'
case m.Last.needImplicitComma(next) && len(m.Stack) != 0: // comma not needed for top-level values
return ','
default:
return 0
}
}
// checkDelim reports whether the specified delimiter should be there given
// the kind of the next token that appears immediately afterwards.
func (m stateMachine) checkDelim(delim byte, next Kind) error {
switch needDelim := m.needDelim(next); {
case needDelim == delim:
return nil
case needDelim == ':':
return errMissingColon
case needDelim == ',':
return errMissingComma
default:
return newInvalidCharacterError([]byte{delim}, "before next token")
}
}
// InvalidateDisabledNamespaces marks all disabled namespaces as invalid.
//
// For efficiency, Marshal and Unmarshal may disable namespaces since there are
// more efficient ways to track duplicate names. However, if an error occurs,
// the namespaces in Encoder or Decoder will be left in an inconsistent state.
// Mark the namespaces as invalid so that future method calls on
// Encoder or Decoder will return an error.
func (m *stateMachine) InvalidateDisabledNamespaces() {
for i := 0; i < m.Depth(); i++ {
e := m.index(i)
if !e.isActiveNamespace() {
e.invalidateNamespace()
}
}
}
// stateEntry encodes several artifacts within a single unsigned integer:
// - whether this represents a JSON object or array,
// - whether this object should check for duplicate names, and
// - how many elements are in this JSON object or array.
type stateEntry uint64
const (
// The type mask (1 bit) records whether this is a JSON object or array.
stateTypeMask stateEntry = 0x8000_0000_0000_0000
stateTypeObject stateEntry = 0x8000_0000_0000_0000
stateTypeArray stateEntry = 0x0000_0000_0000_0000
// The name check mask (2 bit) records whether to update
// the namespaces for the current JSON object and
// whether the namespace is valid.
stateNamespaceMask stateEntry = 0x6000_0000_0000_0000
stateDisableNamespace stateEntry = 0x4000_0000_0000_0000
stateInvalidNamespace stateEntry = 0x2000_0000_0000_0000
// The count mask (61 bits) records the number of elements.
stateCountMask stateEntry = 0x1fff_ffff_ffff_ffff
stateCountLSBMask stateEntry = 0x0000_0000_0000_0001
stateCountOdd stateEntry = 0x0000_0000_0000_0001
stateCountEven stateEntry = 0x0000_0000_0000_0000
)
// Length reports the number of elements in the JSON object or array.
// Each name and value in an object entry is treated as a separate element.
func (e stateEntry) Length() int {
return int(e & stateCountMask)
}
// isObject reports whether this is a JSON object.
func (e stateEntry) isObject() bool {
return e&stateTypeMask == stateTypeObject
}
// isArray reports whether this is a JSON array.
func (e stateEntry) isArray() bool {
return e&stateTypeMask == stateTypeArray
}
// NeedObjectName reports whether the next token must be a JSON string,
// which is necessary for JSON object names.
func (e stateEntry) NeedObjectName() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountEven
}
// needImplicitColon reports whether an colon should occur next,
// which always occurs after JSON object names.
func (e stateEntry) needImplicitColon() bool {
return e.needObjectValue()
}
// needObjectValue reports whether the next token must be a JSON value,
// which is necessary after every JSON object name.
func (e stateEntry) needObjectValue() bool {
return e&(stateTypeMask|stateCountLSBMask) == stateTypeObject|stateCountOdd
}
// needImplicitComma reports whether an comma should occur next,
// which always occurs after a value in a JSON object or array
// before the next value (or name).
func (e stateEntry) needImplicitComma(next Kind) bool {
return !e.needObjectValue() && e.Length() > 0 && next != '}' && next != ']'
}
// Increment increments the number of elements for the current object or array.
// This assumes that overflow won't practically be an issue since
// 1<<bits.OnesCount(stateCountMask) is sufficiently large.
func (e *stateEntry) Increment() {
(*e)++
}
// decrement decrements the number of elements for the current object or array.
// It is the callers responsibility to ensure that e.length > 0.
func (e *stateEntry) decrement() {
(*e)--
}
// DisableNamespace disables the JSON object namespace such that the
// Encoder or Decoder no longer updates the namespace.
func (e *stateEntry) DisableNamespace() {
*e |= stateDisableNamespace
}
// isActiveNamespace reports whether the JSON object namespace is actively
// being updated and used for duplicate name checks.
func (e stateEntry) isActiveNamespace() bool {
return e&(stateDisableNamespace) == 0
}
// invalidateNamespace marks the JSON object namespace as being invalid.
func (e *stateEntry) invalidateNamespace() {
*e |= stateInvalidNamespace
}
// isValidNamespace reports whether the JSON object namespace is valid.
func (e stateEntry) isValidNamespace() bool {
return e&(stateInvalidNamespace) == 0
}
// objectNameStack is a stack of names when descending into a JSON object.
// In contrast to objectNamespaceStack, this only has to remember a single name
// per JSON object.
//
// This data structure may contain offsets to encodeBuffer or decodeBuffer.
// It violates clean abstraction of layers, but is significantly more efficient.
// This ensures that popping and pushing in the common case is a trivial
// push/pop of an offset integer.
//
// The zero value is an empty names stack ready for use.
type objectNameStack struct {
// offsets is a stack of offsets for each name.
// A non-negative offset is the ending offset into the local names buffer.
// A negative offset is the bit-wise inverse of a starting offset into
// a remote buffer (e.g., encodeBuffer or decodeBuffer).
// A math.MinInt offset at the end implies that the last object is empty.
// Invariant: Positive offsets always occur before negative offsets.
offsets []int
// unquotedNames is a back-to-back concatenation of names.
unquotedNames []byte
}
func (ns *objectNameStack) reset() {
ns.offsets = ns.offsets[:0]
ns.unquotedNames = ns.unquotedNames[:0]
if cap(ns.offsets) > 1<<6 {
ns.offsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.unquotedNames) > 1<<10 {
ns.unquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
func (ns *objectNameStack) length() int {
return len(ns.offsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
// It returns an empty string if the last object is empty.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) getUnquoted(i int) []byte {
ns.ensureCopiedBuffer()
if i == 0 {
return ns.unquotedNames[:ns.offsets[0]]
} else {
return ns.unquotedNames[ns.offsets[i-1]:ns.offsets[i-0]]
}
}
// invalidOffset indicates that the last JSON object currently has no name.
const invalidOffset = math.MinInt
// push descends into a nested JSON object.
func (ns *objectNameStack) push() {
ns.offsets = append(ns.offsets, invalidOffset)
}
// ReplaceLastQuotedOffset replaces the last name with the starting offset
// to the quoted name in some remote buffer. All offsets provided must be
// relative to the same buffer until copyQuotedBuffer is called.
func (ns *objectNameStack) ReplaceLastQuotedOffset(i int) {
// Use bit-wise inversion instead of naive multiplication by -1 to avoid
// ambiguity regarding zero (which is a valid offset into the names field).
// Bit-wise inversion is mathematically equivalent to -i-1,
// such that 0 becomes -1, 1 becomes -2, and so forth.
// This ensures that remote offsets are always negative.
ns.offsets[len(ns.offsets)-1] = ^i
}
// replaceLastUnquotedName replaces the last name with the provided name.
//
// Invariant: Must call copyQuotedBuffer beforehand.
func (ns *objectNameStack) replaceLastUnquotedName(s string) {
ns.ensureCopiedBuffer()
var startOffset int
if len(ns.offsets) > 1 {
startOffset = ns.offsets[len(ns.offsets)-2]
}
ns.unquotedNames = append(ns.unquotedNames[:startOffset], s...)
ns.offsets[len(ns.offsets)-1] = len(ns.unquotedNames)
}
// clearLast removes any name in the last JSON object.
// It is semantically equivalent to ns.push followed by ns.pop.
func (ns *objectNameStack) clearLast() {
ns.offsets[len(ns.offsets)-1] = invalidOffset
}
// pop ascends out of a nested JSON object.
func (ns *objectNameStack) pop() {
ns.offsets = ns.offsets[:len(ns.offsets)-1]
}
// copyQuotedBuffer copies names from the remote buffer into the local names
// buffer so that there are no more offset references into the remote buffer.
// This allows the remote buffer to change contents without affecting
// the names that this data structure is trying to remember.
func (ns *objectNameStack) copyQuotedBuffer(b []byte) {
// Find the first negative offset.
var i int
for i = len(ns.offsets) - 1; i >= 0 && ns.offsets[i] < 0; i-- {
continue
}
// Copy each name from the remote buffer into the local buffer.
for i = i + 1; i < len(ns.offsets); i++ {
if i == len(ns.offsets)-1 && ns.offsets[i] == invalidOffset {
if i == 0 {
ns.offsets[i] = 0
} else {
ns.offsets[i] = ns.offsets[i-1]
}
break // last JSON object had a push without any names
}
// As a form of Hyrum proofing, we write an invalid character into the
// buffer to make misuse of Decoder.ReadToken more obvious.
// We need to undo that mutation here.
quotedName := b[^ns.offsets[i]:]
if quotedName[0] == invalidateBufferByte {
quotedName[0] = '"'
}
// Append the unquoted name to the local buffer.
var startOffset int
if i > 0 {
startOffset = ns.offsets[i-1]
}
if n := jsonwire.ConsumeSimpleString(quotedName); n > 0 {
ns.unquotedNames = append(ns.unquotedNames[:startOffset], quotedName[len(`"`):n-len(`"`)]...)
} else {
ns.unquotedNames, _ = jsonwire.AppendUnquote(ns.unquotedNames[:startOffset], quotedName)
}
ns.offsets[i] = len(ns.unquotedNames)
}
}
func (ns *objectNameStack) ensureCopiedBuffer() {
if len(ns.offsets) > 0 && ns.offsets[len(ns.offsets)-1] < 0 {
panic("BUG: copyQuotedBuffer not called beforehand")
}
}
// objectNamespaceStack is a stack of object namespaces.
// This data structure assists in detecting duplicate names.
type objectNamespaceStack []objectNamespace
// reset resets the object namespace stack.
func (nss *objectNamespaceStack) reset() {
if cap(*nss) > 1<<10 {
*nss = nil
}
*nss = (*nss)[:0]
}
// push starts a new namespace for a nested JSON object.
func (nss *objectNamespaceStack) push() {
if cap(*nss) > len(*nss) {
*nss = (*nss)[:len(*nss)+1]
nss.Last().reset()
} else {
*nss = append(*nss, objectNamespace{})
}
}
// Last returns a pointer to the last JSON object namespace.
func (nss objectNamespaceStack) Last() *objectNamespace {
return &nss[len(nss)-1]
}
// pop terminates the namespace for a nested JSON object.
func (nss *objectNamespaceStack) pop() {
*nss = (*nss)[:len(*nss)-1]
}
// objectNamespace is the namespace for a JSON object.
// In contrast to objectNameStack, this needs to remember a all names
// per JSON object.
//
// The zero value is an empty namespace ready for use.
type objectNamespace struct {
// It relies on a linear search over all the names before switching
// to use a Go map for direct lookup.
// endOffsets is a list of offsets to the end of each name in buffers.
// The length of offsets is the number of names in the namespace.
endOffsets []uint
// allUnquotedNames is a back-to-back concatenation of every name in the namespace.
allUnquotedNames []byte
// mapNames is a Go map containing every name in the namespace.
// Only valid if non-nil.
mapNames map[string]struct{}
}
// reset resets the namespace to be empty.
func (ns *objectNamespace) reset() {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
ns.mapNames = nil
if cap(ns.endOffsets) > 1<<6 {
ns.endOffsets = nil // avoid pinning arbitrarily large amounts of memory
}
if cap(ns.allUnquotedNames) > 1<<10 {
ns.allUnquotedNames = nil // avoid pinning arbitrarily large amounts of memory
}
}
// length reports the number of names in the namespace.
func (ns *objectNamespace) length() int {
return len(ns.endOffsets)
}
// getUnquoted retrieves the ith unquoted name in the namespace.
func (ns *objectNamespace) getUnquoted(i int) []byte {
if i == 0 {
return ns.allUnquotedNames[:ns.endOffsets[0]]
} else {
return ns.allUnquotedNames[ns.endOffsets[i-1]:ns.endOffsets[i-0]]
}
}
// lastUnquoted retrieves the last name in the namespace.
func (ns *objectNamespace) lastUnquoted() []byte {
return ns.getUnquoted(ns.length() - 1)
}
// insertQuoted inserts a name and reports whether it was inserted,
// which only occurs if name is not already in the namespace.
// The provided name must be a valid JSON string.
func (ns *objectNamespace) insertQuoted(name []byte, isVerbatim bool) bool {
if isVerbatim {
name = name[len(`"`) : len(name)-len(`"`)]
}
return ns.insert(name, !isVerbatim)
}
func (ns *objectNamespace) InsertUnquoted(name []byte) bool {
return ns.insert(name, false)
}
func (ns *objectNamespace) insert(name []byte, quoted bool) bool {
var allNames []byte
if quoted {
allNames, _ = jsonwire.AppendUnquote(ns.allUnquotedNames, name)
} else {
allNames = append(ns.allUnquotedNames, name...)
}
name = allNames[len(ns.allUnquotedNames):]
// Switch to a map if the buffer is too large for linear search.
// This does not add the current name to the map.
if ns.mapNames == nil && (ns.length() > 64 || len(ns.allUnquotedNames) > 1024) {
ns.mapNames = make(map[string]struct{})
var startOffset uint
for _, endOffset := range ns.endOffsets {
name := ns.allUnquotedNames[startOffset:endOffset]
ns.mapNames[string(name)] = struct{}{} // allocates a new string
startOffset = endOffset
}
}
if ns.mapNames == nil {
// Perform linear search over the buffer to find matching names.
// It provides O(n) lookup, but does not require any allocations.
var startOffset uint
for _, endOffset := range ns.endOffsets {
if string(ns.allUnquotedNames[startOffset:endOffset]) == string(name) {
return false
}
startOffset = endOffset
}
} else {
// Use the map if it is populated.
// It provides O(1) lookup, but requires a string allocation per name.
if _, ok := ns.mapNames[string(name)]; ok {
return false
}
ns.mapNames[string(name)] = struct{}{} // allocates a new string
}
ns.allUnquotedNames = allNames
ns.endOffsets = append(ns.endOffsets, uint(len(ns.allUnquotedNames)))
return true
}
// removeLast removes the last name in the namespace.
func (ns *objectNamespace) removeLast() {
if ns.mapNames != nil {
delete(ns.mapNames, string(ns.lastUnquoted()))
}
if ns.length()-1 == 0 {
ns.endOffsets = ns.endOffsets[:0]
ns.allUnquotedNames = ns.allUnquotedNames[:0]
} else {
ns.endOffsets = ns.endOffsets[:ns.length()-1]
ns.allUnquotedNames = ns.allUnquotedNames[:ns.endOffsets[ns.length()-1]]
}
}
+525
View File
@@ -0,0 +1,525 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"math"
"strconv"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// NOTE: Token is analogous to v1 json.Token.
const (
maxInt64 = math.MaxInt64
minInt64 = math.MinInt64
maxUint64 = math.MaxUint64
minUint64 = 0 // for consistency and readability purposes
invalidTokenPanic = "invalid json.Token; it has been voided by a subsequent json.Decoder call"
)
// Token represents a lexical JSON token, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - a start or end delimiter for a JSON object (i.e., { or } )
// - a start or end delimiter for a JSON array (i.e., [ or ] )
//
// A Token cannot represent entire array or object values, while a [Value] can.
// There is no Token to represent commas and colons since
// these structural tokens can be inferred from the surrounding context.
type Token struct {
nonComparable
// Tokens can exist in either a "raw" or an "exact" form.
// Tokens produced by the Decoder are in the "raw" form.
// Tokens returned by constructors are usually in the "exact" form.
// The Encoder accepts Tokens in either the "raw" or "exact" form.
//
// The following chart shows the possible values for each Token type:
// ╔═════════════════╦════════════╤════════════╤════════════╗
// ║ Token type ║ raw field │ str field │ num field ║
// ╠═════════════════╬════════════╪════════════╪════════════╣
// ║ null (raw) ║ "null" │ "" │ 0 ║
// ║ false (raw) ║ "false" │ "" │ 0 ║
// ║ true (raw) ║ "true" │ "" │ 0 ║
// ║ string (raw) ║ non-empty │ "" │ offset ║
// ║ string (string) ║ nil │ non-empty │ 0 ║
// ║ number (raw) ║ non-empty │ "" │ offset ║
// ║ number (float) ║ nil │ "f" │ non-zero ║
// ║ number (int64) ║ nil │ "i" │ non-zero ║
// ║ number (uint64) ║ nil │ "u" │ non-zero ║
// ║ object (delim) ║ "{" or "}" │ "" │ 0 ║
// ║ array (delim) ║ "[" or "]" │ "" │ 0 ║
// ╚═════════════════╩════════════╧════════════╧════════════╝
//
// Notes:
// - For tokens stored in "raw" form, the num field contains the
// absolute offset determined by raw.previousOffsetStart().
// The buffer itself is stored in raw.previousBuffer().
// - JSON literals and structural characters are always in the "raw" form.
// - JSON strings and numbers can be in either "raw" or "exact" forms.
// - The exact zero value of JSON strings and numbers in the "exact" forms
// have ambiguous representation. Thus, they are always represented
// in the "raw" form.
// raw contains a reference to the raw decode buffer.
// If non-nil, then its value takes precedence over str and num.
// It is only valid if num == raw.previousOffsetStart().
raw *decodeBuffer
// str is the unescaped JSON string if num is zero.
// Otherwise, it is "f", "i", or "u" if num should be interpreted
// as a float64, int64, or uint64, respectively.
str string
// num is a float64, int64, or uint64 stored as a uint64 value.
// It is non-zero for any JSON number in the "exact" form.
num uint64
}
// TODO: Does representing 1-byte delimiters as *decodeBuffer cause performance issues?
var (
Null Token = rawToken("null")
False Token = rawToken("false")
True Token = rawToken("true")
ObjectStart Token = rawToken("{")
ObjectEnd Token = rawToken("}")
ArrayStart Token = rawToken("[")
ArrayEnd Token = rawToken("]")
zeroString Token = rawToken(`""`)
zeroNumber Token = rawToken(`0`)
nanString Token = String("NaN")
pinfString Token = String("Infinity")
ninfString Token = String("-Infinity")
)
func rawToken(s string) Token {
return Token{raw: &decodeBuffer{buf: []byte(s), prevStart: 0, prevEnd: len(s)}}
}
// Bool constructs a Token representing a JSON boolean.
func Bool(b bool) Token {
if b {
return True
}
return False
}
// String constructs a Token representing a JSON string.
// The provided string should contain valid UTF-8, otherwise invalid characters
// may be mangled as the Unicode replacement character.
func String(s string) Token {
if len(s) == 0 {
return zeroString
}
return Token{str: s}
}
// Float constructs a Token representing a JSON number.
// The values NaN, +Inf, and -Inf will be represented
// as a JSON string with the values "NaN", "Infinity", and "-Infinity".
func Float(n float64) Token {
switch {
case math.Float64bits(n) == 0:
return zeroNumber
case math.IsNaN(n):
return nanString
case math.IsInf(n, +1):
return pinfString
case math.IsInf(n, -1):
return ninfString
}
return Token{str: "f", num: math.Float64bits(n)}
}
// Int constructs a Token representing a JSON number from an int64.
func Int(n int64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "i", num: uint64(n)}
}
// Uint constructs a Token representing a JSON number from a uint64.
func Uint(n uint64) Token {
if n == 0 {
return zeroNumber
}
return Token{str: "u", num: uint64(n)}
}
// Clone makes a copy of the Token such that its value remains valid
// even after a subsequent [Decoder.Read] call.
func (t Token) Clone() Token {
// TODO: Allow caller to avoid any allocations?
if raw := t.raw; raw != nil {
// Avoid copying globals.
if t.raw.prevStart == 0 {
switch t.raw {
case Null.raw:
return Null
case False.raw:
return False
case True.raw:
return True
case ObjectStart.raw:
return ObjectStart
case ObjectEnd.raw:
return ObjectEnd
case ArrayStart.raw:
return ArrayStart
case ArrayEnd.raw:
return ArrayEnd
}
}
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
// TODO(https://go.dev/issue/45038): Use bytes.Clone.
buf := append([]byte(nil), raw.PreviousBuffer()...)
return Token{raw: &decodeBuffer{buf: buf, prevStart: 0, prevEnd: len(buf)}}
}
return t
}
// Bool returns the value for a JSON boolean.
// It panics if the token kind is not a JSON boolean.
func (t Token) Bool() bool {
switch t.raw {
case True.raw:
return true
case False.raw:
return false
default:
panic("invalid JSON token kind: " + t.Kind().String())
}
}
// appendString appends a JSON string to dst and returns it.
// It panics if t is not a JSON string.
func (t Token) appendString(dst []byte, flags *jsonflags.Flags) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw string value.
buf := raw.PreviousBuffer()
if Kind(buf[0]) == '"' {
if jsonwire.ConsumeSimpleString(buf) == len(buf) {
return append(dst, buf...), nil
}
dst, _, err := jsonwire.ReformatString(dst, buf, flags)
return dst, err
}
} else if len(t.str) != 0 && t.num == 0 {
// Handle exact string value.
return jsonwire.AppendQuote(dst, t.str, flags)
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// String returns the unescaped string value for a JSON string.
// For other JSON kinds, this returns the raw JSON representation.
func (t Token) String() string {
// This is inlinable to take advantage of "function outlining".
// This avoids an allocation for the string(b) conversion
// if the caller does not use the string in an escaping manner.
// See https://blog.filippo.io/efficient-go-apis-with-the-inliner/
s, b := t.string()
if len(b) > 0 {
return string(b)
}
return s
}
func (t Token) string() (string, []byte) {
if raw := t.raw; raw != nil {
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.PreviousBuffer()
if buf[0] == '"' {
// TODO: Preserve ValueFlags in Token?
isVerbatim := jsonwire.ConsumeSimpleString(buf) == len(buf)
return "", jsonwire.UnquoteMayCopy(buf, isVerbatim)
}
// Handle tokens that are not JSON strings for fmt.Stringer.
return "", buf
}
if len(t.str) != 0 && t.num == 0 {
return t.str, nil
}
// Handle tokens that are not JSON strings for fmt.Stringer.
if t.num > 0 {
switch t.str[0] {
case 'f':
return string(jsonwire.AppendFloat(nil, math.Float64frombits(t.num), 64)), nil
case 'i':
return strconv.FormatInt(int64(t.num), 10), nil
case 'u':
return strconv.FormatUint(uint64(t.num), 10), nil
}
}
return "<invalid json.Token>", nil
}
// appendNumber appends a JSON number to dst and returns it.
// It panics if t is not a JSON number.
func (t Token) appendNumber(dst []byte, canonicalize bool) ([]byte, error) {
if raw := t.raw; raw != nil {
// Handle raw number value.
buf := raw.PreviousBuffer()
if Kind(buf[0]).normalize() == '0' {
if !canonicalize {
return append(dst, buf...), nil
}
dst, _, err := jsonwire.ReformatNumber(dst, buf, canonicalize)
return dst, err
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return jsonwire.AppendFloat(dst, math.Float64frombits(t.num), 64), nil
case 'i':
return strconv.AppendInt(dst, int64(t.num), 10), nil
case 'u':
return strconv.AppendUint(dst, uint64(t.num), 10), nil
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Float returns the floating-point value for a JSON number.
// It returns a NaN, +Inf, or -Inf value for any JSON string
// with the values "NaN", "Infinity", or "-Infinity".
// It panics for all other cases.
func (t Token) Float() float64 {
if raw := t.raw; raw != nil {
// Handle raw number value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
buf := raw.PreviousBuffer()
if Kind(buf[0]).normalize() == '0' {
fv, _ := jsonwire.ParseFloat(buf, 64)
return fv
}
} else if t.num != 0 {
// Handle exact number value.
switch t.str[0] {
case 'f':
return math.Float64frombits(t.num)
case 'i':
return float64(int64(t.num))
case 'u':
return float64(uint64(t.num))
}
}
// Handle string values with "NaN", "Infinity", or "-Infinity".
if t.Kind() == '"' {
switch t.String() {
case "NaN":
return math.NaN()
case "Infinity":
return math.Inf(+1)
case "-Infinity":
return math.Inf(-1)
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Int returns the signed integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an int64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Int() int64 {
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.PreviousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if numAbs, ok := jsonwire.ParseUint(buf); ok {
if neg {
if numAbs > -minInt64 {
return minInt64
}
return -1 * int64(numAbs)
} else {
if numAbs > +maxInt64 {
return maxInt64
}
return +1 * int64(numAbs)
}
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'i':
return int64(t.num)
case 'u':
if t.num > maxInt64 {
return maxInt64
}
return int64(t.num)
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxInt64:
return maxInt64
case fv <= minInt64:
return minInt64
default:
return int64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Uint returns the unsigned integer value for a JSON number.
// The fractional component of any number is ignored (truncation toward zero).
// Any number beyond the representation of an uint64 will be saturated
// to the closest representable value.
// It panics if the token kind is not a JSON number.
func (t Token) Uint() uint64 {
// NOTE: This accessor returns 0 for any negative JSON number,
// which might be surprising, but is at least consistent with the behavior
// of saturating out-of-bounds numbers to the closest representable number.
if raw := t.raw; raw != nil {
// Handle raw integer value.
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
neg := false
buf := raw.PreviousBuffer()
if len(buf) > 0 && buf[0] == '-' {
neg, buf = true, buf[1:]
}
if num, ok := jsonwire.ParseUint(buf); ok {
if neg {
return minUint64
}
return num
}
} else if t.num != 0 {
// Handle exact integer value.
switch t.str[0] {
case 'u':
return t.num
case 'i':
if int64(t.num) < minUint64 {
return minUint64
}
return uint64(int64(t.num))
}
}
// Handle JSON number that is a floating-point value.
if t.Kind() == '0' {
switch fv := t.Float(); {
case fv >= maxUint64:
return maxUint64
case fv <= minUint64:
return minUint64
default:
return uint64(fv) // truncation toward zero
}
}
panic("invalid JSON token kind: " + t.Kind().String())
}
// Kind returns the token kind.
func (t Token) Kind() Kind {
switch {
case t.raw != nil:
raw := t.raw
if uint64(raw.previousOffsetStart()) != t.num {
panic(invalidTokenPanic)
}
return Kind(t.raw.buf[raw.prevStart]).normalize()
case t.num != 0:
return '0'
case len(t.str) != 0:
return '"'
default:
return invalidKind
}
}
// Kind represents each possible JSON token kind with a single byte,
// which is conveniently the first byte of that kind's grammar
// with the restriction that numbers always be represented with '0':
//
// - 'n': null
// - 'f': false
// - 't': true
// - '"': string
// - '0': number
// - '{': object start
// - '}': object end
// - '[': array start
// - ']': array end
//
// An invalid kind is usually represented using 0,
// but may be non-zero due to invalid JSON data.
type Kind byte
const invalidKind Kind = 0
// String prints the kind in a humanly readable fashion.
func (k Kind) String() string {
switch k {
case 'n':
return "null"
case 'f':
return "false"
case 't':
return "true"
case '"':
return "string"
case '0':
return "number"
case '{':
return "{"
case '}':
return "}"
case '[':
return "["
case ']':
return "]"
default:
return "<invalid json.Kind: " + jsonwire.QuoteRune(string(k)) + ">"
}
}
// normalize coalesces all possible starting characters of a number as just '0'.
func (k Kind) normalize() Kind {
if k == '-' || ('0' <= k && k <= '9') {
return '0'
}
return k
}
+305
View File
@@ -0,0 +1,305 @@
// Copyright 2020 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.
package jsontext
import (
"bytes"
"errors"
"io"
"slices"
"strings"
"sync"
"github.com/go-json-experiment/json/internal/jsonflags"
"github.com/go-json-experiment/json/internal/jsonwire"
)
// NOTE: Value is analogous to v1 json.RawMessage.
// Value represents a single raw JSON value, which may be one of the following:
// - a JSON literal (i.e., null, true, or false)
// - a JSON string (e.g., "hello, world!")
// - a JSON number (e.g., 123.456)
// - an entire JSON object (e.g., {"fizz":"buzz"} )
// - an entire JSON array (e.g., [1,2,3] )
//
// Value can represent entire array or object values, while [Token] cannot.
// Value may contain leading and/or trailing whitespace.
type Value []byte
// Clone returns a copy of v.
func (v Value) Clone() Value {
return bytes.Clone(v)
}
// String returns the string formatting of v.
func (v Value) String() string {
if v == nil {
return "null"
}
return string(v)
}
// IsValid reports whether the raw JSON value is syntactically valid
// according to RFC 7493.
//
// It verifies whether the input is properly encoded as UTF-8,
// that escape sequences within strings decode to valid Unicode codepoints, and
// that all names in each object are unique.
// It does not verify whether numbers are representable within the limits
// of any common numeric type (e.g., float64, int64, or uint64).
func (v Value) IsValid() bool {
d := getBufferedDecoder(v)
defer putBufferedDecoder(d)
_, errVal := d.ReadValue()
_, errEOF := d.ReadToken()
return errVal == nil && errEOF == io.EOF
}
// Compact removes all whitespace from the raw JSON value.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already compacted, then the buffer is not mutated.
func (v *Value) Compact() error {
return v.reformat(false, false, "", "")
}
// Indent reformats the whitespace in the raw JSON value so that each element
// in a JSON object or array begins on a new, indented line beginning with
// prefix followed by one or more copies of indent according to the nesting.
// The value does not begin with the prefix nor any indention,
// to make it easier to embed inside other formatted JSON data.
//
// It does not reformat JSON strings to use any other representation.
// It is guaranteed to succeed if the input is valid.
// If the value is already indented properly, then the buffer is not mutated.
//
// The prefix and indent strings must be composed of only spaces and/or tabs.
func (v *Value) Indent(prefix, indent string) error {
return v.reformat(false, true, prefix, indent)
}
// Canonicalize canonicalizes the raw JSON value according to the
// JSON Canonicalization Scheme (JCS) as defined by RFC 8785
// where it produces a stable representation of a JSON value.
//
// The output stability is dependent on the stability of the application data
// (see RFC 8785, Appendix E). It cannot produce stable output from
// fundamentally unstable input. For example, if the JSON value
// contains ephemeral data (e.g., a frequently changing timestamp),
// then the value is still unstable regardless of whether this is called.
//
// Note that JCS treats all JSON numbers as IEEE 754 double precision numbers.
// Any numbers with precision beyond what is representable by that form
// will lose their precision when canonicalized. For example, integer values
// beyond ±2⁵³ will lose their precision. It is recommended that
// int64 and uint64 data types be represented as a JSON string.
//
// It is guaranteed to succeed if the input is valid.
// If the value is already canonicalized, then the buffer is not mutated.
func (v *Value) Canonicalize() error {
return v.reformat(true, false, "", "")
}
// TODO: Instead of implementing the v1 Marshaler/Unmarshaler,
// consider implementing the v2 versions instead.
// MarshalJSON returns v as the JSON encoding of v.
// It returns the stored value as the raw JSON output without any validation.
// If v is nil, then this returns a JSON null.
func (v Value) MarshalJSON() ([]byte, error) {
// NOTE: This matches the behavior of v1 json.RawMessage.MarshalJSON.
if v == nil {
return []byte("null"), nil
}
return v, nil
}
// UnmarshalJSON sets v as the JSON encoding of b.
// It stores a copy of the provided raw JSON input without any validation.
func (v *Value) UnmarshalJSON(b []byte) error {
// NOTE: This matches the behavior of v1 json.RawMessage.UnmarshalJSON.
if v == nil {
return errors.New("json.Value: UnmarshalJSON on nil pointer")
}
*v = append((*v)[:0], b...)
return nil
}
// Kind returns the starting token kind.
// For a valid value, this will never include '}' or ']'.
func (v Value) Kind() Kind {
if v := v[jsonwire.ConsumeWhitespace(v):]; len(v) > 0 {
return Kind(v[0]).normalize()
}
return invalidKind
}
func (v *Value) reformat(canonical, multiline bool, prefix, indent string) error {
// Write the entire value to reformat all tokens and whitespace.
e := getBufferedEncoder()
defer putBufferedEncoder(e)
eo := &e.s.Struct
if canonical {
eo.Flags.Set(jsonflags.AllowInvalidUTF8 | 0) // per RFC 8785, section 3.2.4
eo.Flags.Set(jsonflags.AllowDuplicateNames | 0) // per RFC 8785, section 3.1
eo.Flags.Set(jsonflags.CanonicalizeNumbers | 1) // per RFC 8785, section 3.2.2.3
eo.Flags.Set(jsonflags.PreserveRawStrings | 0) // per RFC 8785, section 3.2.2.2
eo.Flags.Set(jsonflags.EscapeForHTML | 0) // per RFC 8785, section 3.2.2.2
eo.Flags.Set(jsonflags.EscapeForJS | 0) // per RFC 8785, section 3.2.2.2
eo.Flags.Set(jsonflags.Expand | 0) // per RFC 8785, section 3.2.1
} else {
if s := strings.TrimLeft(prefix, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent prefix")
}
if s := strings.TrimLeft(indent, " \t"); len(s) > 0 {
panic("json: invalid character " + jsonwire.QuoteRune(s) + " in indent")
}
eo.Flags.Set(jsonflags.AllowInvalidUTF8 | 1)
eo.Flags.Set(jsonflags.AllowDuplicateNames | 1)
eo.Flags.Set(jsonflags.PreserveRawStrings | 1)
if multiline {
eo.Flags.Set(jsonflags.Expand | 1)
eo.Flags.Set(jsonflags.Indent | 1)
eo.Flags.Set(jsonflags.IndentPrefix | 1)
eo.IndentPrefix = prefix
eo.Indent = indent
} else {
eo.Flags.Set(jsonflags.Expand | 0)
}
}
eo.Flags.Set(jsonflags.OmitTopLevelNewline | 1)
if err := e.s.WriteValue(*v); err != nil {
return err
}
// For canonical output, we may need to reorder object members.
if canonical {
// Obtain a buffered encoder just to use its internal buffer as
// a scratch buffer in reorderObjects for reordering object members.
e2 := getBufferedEncoder()
defer putBufferedEncoder(e2)
// Disable redundant checks performed earlier during encoding.
d := getBufferedDecoder(e.s.Buf)
defer putBufferedDecoder(d)
d.s.Flags.Set(jsonflags.AllowDuplicateNames | jsonflags.AllowInvalidUTF8 | 1)
reorderObjects(d, &e2.s.Buf) // per RFC 8785, section 3.2.3
}
// Store the result back into the value if different.
if !bytes.Equal(*v, e.s.Buf) {
*v = append((*v)[:0], e.s.Buf...)
}
return nil
}
type memberName struct {
// name is the unescaped name.
name []byte
// before and after are byte offsets into Decoder.buf that represents
// the entire name/value pair. It may contain leading commas.
before, after int64
}
var memberNamePool = sync.Pool{New: func() any { return new([]memberName) }}
func getMemberNames() *[]memberName {
ns := memberNamePool.Get().(*[]memberName)
*ns = (*ns)[:0]
return ns
}
func putMemberNames(ns *[]memberName) {
if cap(*ns) < 1<<10 {
clear(*ns) // avoid pinning name
memberNamePool.Put(ns)
}
}
// reorderObjects recursively reorders all object members in place
// according to the ordering specified in RFC 8785, section 3.2.3.
//
// Pre-conditions:
// - The value is valid (i.e., no decoder errors should ever occur).
// - The value is compact (i.e., no whitespace is present).
// - Initial call is provided a Decoder reading from the start of v.
//
// Post-conditions:
// - Exactly one JSON value is read from the Decoder.
// - All fully-parsed JSON objects are reordered by directly moving
// the members in the value buffer.
//
// The runtime is approximately O(n·log(n)) + O(m·log(m)),
// where n is len(v) and m is the total number of object members.
func reorderObjects(d *Decoder, scratch *[]byte) {
switch tok, _ := d.ReadToken(); tok.Kind() {
case '{':
// Iterate and collect the name and offsets for every object member.
members := getMemberNames()
defer putMemberNames(members)
var prevName []byte
isSorted := true
beforeBody := d.InputOffset() // offset after '{'
for d.PeekKind() != '}' {
beforeName := d.InputOffset()
var flags jsonwire.ValueFlags
name, _ := d.s.ReadValue(&flags)
name = jsonwire.UnquoteMayCopy(name, flags.IsVerbatim())
reorderObjects(d, scratch)
afterValue := d.InputOffset()
if isSorted && len(*members) > 0 {
isSorted = jsonwire.CompareUTF16(prevName, []byte(name)) < 0
}
*members = append(*members, memberName{name, beforeName, afterValue})
prevName = name
}
afterBody := d.InputOffset() // offset before '}'
d.ReadToken()
// Sort the members; return early if it's already sorted.
if isSorted {
return
}
slices.SortFunc(*members, func(x, y memberName) int {
return jsonwire.CompareUTF16(x.name, y.name)
})
// Append the reordered members to a new buffer,
// then copy the reordered members back over the original members.
// Avoid swapping in place since each member may be a different size
// where moving a member over a smaller member may corrupt the data
// for subsequent members before they have been moved.
//
// The following invariant must hold:
// sum([m.after-m.before for m in members]) == afterBody-beforeBody
sorted := (*scratch)[:0]
for i, member := range *members {
if d.s.buf[member.before] == ',' {
member.before++ // trim leading comma
}
sorted = append(sorted, d.s.buf[member.before:member.after]...)
if i < len(*members)-1 {
sorted = append(sorted, ',') // append trailing comma
}
}
if int(afterBody-beforeBody) != len(sorted) {
panic("BUG: length invariant violated")
}
copy(d.s.buf[beforeBody:afterBody], sorted)
// Update scratch buffer to the largest amount ever used.
if len(sorted) > len(*scratch) {
*scratch = sorted
}
case '[':
for d.PeekKind() != ']' {
reorderObjects(d, scratch)
}
d.ReadToken()
}
}
+48 -3
View File
@@ -37,6 +37,7 @@ var (
ChainHookOutput *ChainHook = ChainHookRef(unix.NF_INET_LOCAL_OUT)
ChainHookPostrouting *ChainHook = ChainHookRef(unix.NF_INET_POST_ROUTING)
ChainHookIngress *ChainHook = ChainHookRef(unix.NF_NETDEV_INGRESS)
ChainHookEgress *ChainHook = ChainHookRef(unix.NF_NETDEV_EGRESS)
)
// ChainHookRef returns a pointer to a ChainHookRef value.
@@ -101,6 +102,7 @@ type Chain struct {
Priority *ChainPriority
Type ChainType
Policy *ChainPolicy
Device string
}
// AddChain adds the specified Chain. See also
@@ -118,6 +120,11 @@ func (cc *Conn) AddChain(c *Chain) *Chain {
{Type: unix.NFTA_HOOK_HOOKNUM, Data: binaryutil.BigEndian.PutUint32(uint32(*c.Hooknum))},
{Type: unix.NFTA_HOOK_PRIORITY, Data: binaryutil.BigEndian.PutUint32(uint32(*c.Priority))},
}
if c.Device != "" {
hookAttr = append(hookAttr, netlink.Attribute{Type: unix.NFTA_HOOK_DEV, Data: []byte(c.Device + "\x00")})
}
data = append(data, cc.marshalAttr([]netlink.Attribute{
{Type: unix.NLA_F_NESTED | unix.NFTA_CHAIN_HOOK, Data: cc.marshalAttr(hookAttr)},
})...)
@@ -186,6 +193,43 @@ func (cc *Conn) ListChains() ([]*Chain, error) {
return cc.ListChainsOfTableFamily(TableFamilyUnspecified)
}
// ListChain returns a single chain configured in the specified table
func (cc *Conn) ListChain(table *Table, chain string) (*Chain, error) {
conn, closer, err := cc.netlinkConn()
if err != nil {
return nil, err
}
defer func() { _ = closer() }()
attrs := []netlink.Attribute{
{Type: unix.NFTA_TABLE_NAME, Data: []byte(table.Name + "\x00")},
{Type: unix.NFTA_CHAIN_NAME, Data: []byte(chain + "\x00")},
}
msg := netlink.Message{
Header: netlink.Header{
Type: netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_GETCHAIN),
Flags: netlink.Request,
},
Data: append(extraHeader(uint8(table.Family), 0), cc.marshalAttr(attrs)...),
}
response, err := conn.Execute(msg)
if err != nil {
return nil, fmt.Errorf("conn.Execute failed: %v", err)
}
if got, want := len(response), 1; got != want {
return nil, fmt.Errorf("expected %d response message for chain, got %d", want, got)
}
ch, err := chainFromMsg(response[0])
if err != nil {
return nil, err
}
return ch, nil
}
// ListChainsOfTableFamily returns currently configured chains for the specified
// family in the kernel. It lists all chains ins all tables if family is
// TableFamilyUnspecified.
@@ -223,9 +267,10 @@ func (cc *Conn) ListChainsOfTableFamily(family TableFamily) ([]*Chain, error) {
}
func chainFromMsg(msg netlink.Message) (*Chain, error) {
chainHeaderType := netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN)
if got, want := msg.Header.Type, chainHeaderType; got != want {
return nil, fmt.Errorf("unexpected header type: got %v, want %v", got, want)
newChainHeaderType := netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWCHAIN)
delChainHeaderType := netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELCHAIN)
if got, want1, want2 := msg.Header.Type, newChainHeaderType, delChainHeaderType; got != want1 && got != want2 {
return nil, fmt.Errorf("unexpected header type: got %v, want %v or %v", got, want1, want2)
}
var c Chain
+33 -7
View File
@@ -17,6 +17,7 @@ package nftables
import (
"errors"
"fmt"
"os"
"sync"
"github.com/google/nftables/binaryutil"
@@ -154,14 +155,33 @@ func receiveAckAware(nlconn *netlink.Conn, sentMsgFlags netlink.HeaderFlags) ([]
return reply, nil
}
// Dump flag is not set, we expect an ack
if len(reply) != 0 {
last := reply[len(reply)-1]
for re := last.Header.Type; (re&netlink.Overrun) == netlink.Overrun && (re&netlink.Done) != netlink.Done; re = last.Header.Type {
// we are not finished, the message is overrun
r, err := nlconn.Receive()
if err != nil {
return nil, err
}
reply = append(reply, r...)
last = reply[len(reply)-1]
}
if last.Header.Type == netlink.Error && binaryutil.BigEndian.Uint32(last.Data[:4]) == 0 {
// we have already collected an ack
return reply, nil
}
}
// Now we expect an ack
ack, err := nlconn.Receive()
if err != nil {
return nil, err
}
if len(ack) == 0 {
return nil, errors.New("received an empty ack")
// received an empty ack?
return reply, nil
}
msg := ack[0]
@@ -230,16 +250,22 @@ func (cc *Conn) Flush() error {
return fmt.Errorf("SendMessages: %w", err)
}
var errs error
// Fetch the requested acknowledgement for each message we sent.
for _, msg := range cc.messages {
if msg.Header.Flags&netlink.Acknowledge == 0 {
continue // message did not request an acknowledgement
}
if _, err := conn.Receive(); err != nil {
return fmt.Errorf("conn.Receive: %w", err)
if _, err := receiveAckAware(conn, msg.Header.Flags); err != nil {
if errors.Is(err, os.ErrPermission) {
// Kernel will only send one permission error to user space.
return err
}
errs = errors.Join(errs, err)
}
}
if errs != nil {
return fmt.Errorf("conn.Receive: %w", errs)
}
return nil
}
+1 -2
View File
@@ -57,11 +57,10 @@ func (c *CounterObj) marshal(data bool) ([]byte, error) {
if err != nil {
return nil, err
}
const NFT_OBJECT_COUNTER = 1 // TODO: get into x/sys/unix
attrs := []netlink.Attribute{
{Type: unix.NFTA_OBJ_TABLE, Data: []byte(c.Table.Name + "\x00")},
{Type: unix.NFTA_OBJ_NAME, Data: []byte(c.Name + "\x00")},
{Type: unix.NFTA_OBJ_TYPE, Data: binaryutil.BigEndian.PutUint32(NFT_OBJECT_COUNTER)},
{Type: unix.NFTA_OBJ_TYPE, Data: binaryutil.BigEndian.PutUint32(unix.NFT_OBJECT_COUNTER)},
}
if data {
attrs = append(attrs, netlink.Attribute{Type: unix.NLA_F_NESTED | unix.NFTA_OBJ_DATA, Data: obj})
+7 -4
View File
@@ -134,6 +134,8 @@ func exprsFromBytes(fam byte, ad *netlink.AttributeDecoder, b []byte) ([]Any, er
e = &Reject{}
case "masq":
e = &Masq{}
case "hash":
e = &Hash{}
}
if e == nil {
// TODO: introduce an opaque expression type so that users know
@@ -277,14 +279,15 @@ type Masq struct {
RegProtoMax uint32
}
// TODO, Once the constants below are available in golang.org/x/sys/unix, switch to use those.
const (
// NF_NAT_RANGE_PROTO_RANDOM defines flag for a random masquerade
NF_NAT_RANGE_PROTO_RANDOM = 0x4
NF_NAT_RANGE_PROTO_RANDOM = unix.NF_NAT_RANGE_PROTO_RANDOM
// NF_NAT_RANGE_PROTO_RANDOM_FULLY defines flag for a fully random masquerade
NF_NAT_RANGE_PROTO_RANDOM_FULLY = 0x10
NF_NAT_RANGE_PROTO_RANDOM_FULLY = unix.NF_NAT_RANGE_PROTO_RANDOM_FULLY
// NF_NAT_RANGE_PERSISTENT defines flag for a persistent masquerade
NF_NAT_RANGE_PERSISTENT = 0x8
NF_NAT_RANGE_PERSISTENT = unix.NF_NAT_RANGE_PERSISTENT
// NF_NAT_RANGE_PREFIX defines flag for a prefix masquerade
NF_NAT_RANGE_PREFIX = unix.NF_NAT_RANGE_NETMAP
)
func (e *Masq) marshal(fam byte) ([]byte, error) {
+10 -3
View File
@@ -41,15 +41,22 @@ type Hash struct {
}
func (e *Hash) marshal(fam byte) ([]byte, error) {
data, err := netlink.MarshalAttributes([]netlink.Attribute{
hashAttrs := []netlink.Attribute{
{Type: unix.NFTA_HASH_SREG, Data: binaryutil.BigEndian.PutUint32(uint32(e.SourceRegister))},
{Type: unix.NFTA_HASH_DREG, Data: binaryutil.BigEndian.PutUint32(uint32(e.DestRegister))},
{Type: unix.NFTA_HASH_LEN, Data: binaryutil.BigEndian.PutUint32(uint32(e.Length))},
{Type: unix.NFTA_HASH_MODULUS, Data: binaryutil.BigEndian.PutUint32(uint32(e.Modulus))},
{Type: unix.NFTA_HASH_SEED, Data: binaryutil.BigEndian.PutUint32(uint32(e.Seed))},
}
if e.Seed != 0 {
hashAttrs = append(hashAttrs, netlink.Attribute{
Type: unix.NFTA_HASH_SEED, Data: binaryutil.BigEndian.PutUint32(uint32(e.Seed)),
})
}
hashAttrs = append(hashAttrs, []netlink.Attribute{
{Type: unix.NFTA_HASH_OFFSET, Data: binaryutil.BigEndian.PutUint32(uint32(e.Offset))},
{Type: unix.NFTA_HASH_TYPE, Data: binaryutil.BigEndian.PutUint32(uint32(e.Type))},
})
}...)
data, err := netlink.MarshalAttributes(hashAttrs)
if err != nil {
return nil, err
}
+5
View File
@@ -40,6 +40,7 @@ type NAT struct {
Random bool
FullyRandom bool
Persistent bool
Prefix bool
}
// |00048|N-|00001| |len |flags| type|
@@ -82,6 +83,9 @@ func (e *NAT) marshal(fam byte) ([]byte, error) {
if e.Persistent {
flags |= NF_NAT_RANGE_PERSISTENT
}
if e.Prefix {
flags |= NF_NAT_RANGE_PREFIX
}
if flags != 0 {
attrs = append(attrs, netlink.Attribute{Type: unix.NFTA_NAT_FLAGS, Data: binaryutil.BigEndian.PutUint32(flags)})
}
@@ -121,6 +125,7 @@ func (e *NAT) unmarshal(fam byte, data []byte) error {
e.Persistent = (flags & NF_NAT_RANGE_PERSISTENT) != 0
e.Random = (flags & NF_NAT_RANGE_PROTO_RANDOM) != 0
e.FullyRandom = (flags & NF_NAT_RANGE_PROTO_RANDOM_FULLY) != 0
e.Prefix = (flags & NF_NAT_RANGE_PREFIX) != 0
}
}
return ad.Err()
+3 -4
View File
@@ -32,10 +32,9 @@ const (
QueueTotal QueueAttribute = unix.NFTA_QUEUE_TOTAL
QueueFlags QueueAttribute = unix.NFTA_QUEUE_FLAGS
// TODO: get into x/sys/unix
QueueFlagBypass QueueFlag = 0x01
QueueFlagFanout QueueFlag = 0x02
QueueFlagMask QueueFlag = 0x03
QueueFlagBypass QueueFlag = unix.NFT_QUEUE_FLAG_BYPASS
QueueFlagFanout QueueFlag = unix.NFT_QUEUE_FLAG_CPU_FANOUT
QueueFlagMask QueueFlag = unix.NFT_QUEUE_FLAG_MASK
)
type Queue struct {
+89
View File
@@ -0,0 +1,89 @@
// Copyright 2023 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package expr
import (
"encoding/binary"
"golang.org/x/sys/unix"
"github.com/google/nftables/binaryutil"
"github.com/mdlayher/netlink"
)
type Socket struct {
Key SocketKey
Level uint32
Register uint32
}
type SocketKey uint32
const (
// TODO, Once the constants below are available in golang.org/x/sys/unix, switch to use those.
NFTA_SOCKET_KEY = 1
NFTA_SOCKET_DREG = 2
NFTA_SOCKET_LEVEL = 3
NFT_SOCKET_TRANSPARENT = 0
NFT_SOCKET_MARK = 1
NFT_SOCKET_WILDCARD = 2
NFT_SOCKET_CGROUPV2 = 3
SocketKeyTransparent SocketKey = NFT_SOCKET_TRANSPARENT
SocketKeyMark SocketKey = NFT_SOCKET_MARK
SocketKeyWildcard SocketKey = NFT_SOCKET_WILDCARD
SocketKeyCgroupv2 SocketKey = NFT_SOCKET_CGROUPV2
)
func (e *Socket) marshal(fam byte) ([]byte, error) {
// NOTE: Socket.Level is only used when Socket.Key == SocketKeyCgroupv2. But `nft` always encoding it. Check link below:
// http://git.netfilter.org/nftables/tree/src/netlink_linearize.c?id=0583bac241ea18c9d7f61cb20ca04faa1e043b78#n319
exprData, err := netlink.MarshalAttributes(
[]netlink.Attribute{
{Type: NFTA_SOCKET_DREG, Data: binaryutil.BigEndian.PutUint32(e.Register)},
{Type: NFTA_SOCKET_KEY, Data: binaryutil.BigEndian.PutUint32(uint32(e.Key))},
{Type: NFTA_SOCKET_LEVEL, Data: binaryutil.BigEndian.PutUint32(uint32(e.Level))},
},
)
if err != nil {
return nil, err
}
return netlink.MarshalAttributes([]netlink.Attribute{
{Type: unix.NFTA_EXPR_NAME, Data: []byte("socket\x00")},
{Type: unix.NLA_F_NESTED | unix.NFTA_EXPR_DATA, Data: exprData},
})
}
func (e *Socket) unmarshal(fam byte, data []byte) error {
ad, err := netlink.NewAttributeDecoder(data)
if err != nil {
return err
}
ad.ByteOrder = binary.BigEndian
for ad.Next() {
switch ad.Type() {
case NFTA_SOCKET_DREG:
e.Register = ad.Uint32()
case NFTA_SOCKET_KEY:
e.Key = SocketKey(ad.Uint32())
case NFTA_SOCKET_LEVEL:
e.Level = ad.Uint32()
}
}
return ad.Err()
}
+20 -6
View File
@@ -25,22 +25,34 @@ import (
const (
// NFTA_TPROXY_FAMILY defines attribute for a table family
NFTA_TPROXY_FAMILY = 0x01
// NFTA_TPROXY_REG defines attribute for a register carrying redirection port value
NFTA_TPROXY_REG = 0x03
// NFTA_TPROXY_REG_ADDR defines attribute for a register carrying redirection address value
NFTA_TPROXY_REG_ADDR = 0x02
// NFTA_TPROXY_REG_PORT defines attribute for a register carrying redirection port value
NFTA_TPROXY_REG_PORT = 0x03
)
// TProxy defines struct with parameters for the transparent proxy
type TProxy struct {
Family byte
TableFamily byte
RegAddr uint32
RegPort uint32
}
func (e *TProxy) marshal(fam byte) ([]byte, error) {
data, err := netlink.MarshalAttributes([]netlink.Attribute{
attrs := []netlink.Attribute{
{Type: NFTA_TPROXY_FAMILY, Data: binaryutil.BigEndian.PutUint32(uint32(e.Family))},
{Type: NFTA_TPROXY_REG, Data: binaryutil.BigEndian.PutUint32(e.RegPort)},
})
{Type: NFTA_TPROXY_REG_PORT, Data: binaryutil.BigEndian.PutUint32(e.RegPort)},
}
if e.RegAddr != 0 {
attrs = append(attrs, netlink.Attribute{
Type: NFTA_TPROXY_REG_ADDR,
Data: binaryutil.BigEndian.PutUint32(e.RegAddr),
})
}
data, err := netlink.MarshalAttributes(attrs)
if err != nil {
return nil, err
}
@@ -60,8 +72,10 @@ func (e *TProxy) unmarshal(fam byte, data []byte) error {
switch ad.Type() {
case NFTA_TPROXY_FAMILY:
e.Family = ad.Uint8()
case NFTA_TPROXY_REG:
case NFTA_TPROXY_REG_PORT:
e.RegPort = ad.Uint32()
case NFTA_TPROXY_REG_ADDR:
e.RegAddr = ad.Uint32()
}
}
return ad.Err()
+319
View File
@@ -0,0 +1,319 @@
// Copyright 2018 Google LLC. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package nftables
import (
"math"
"strings"
"sync"
"github.com/mdlayher/netlink"
"golang.org/x/sys/unix"
)
type MonitorAction uint8
// Possible MonitorAction values.
const (
MonitorActionNew MonitorAction = 1 << iota
MonitorActionDel
MonitorActionMask MonitorAction = (1 << iota) - 1
MonitorActionAny MonitorAction = MonitorActionMask
)
type MonitorObject uint32
// Possible MonitorObject values.
const (
MonitorObjectTables MonitorObject = 1 << iota
MonitorObjectChains
MonitorObjectSets
MonitorObjectRules
MonitorObjectElements
MonitorObjectRuleset
MonitorObjectMask MonitorObject = (1 << iota) - 1
MonitorObjectAny MonitorObject = MonitorObjectMask
)
var (
monitorFlags = map[MonitorAction]map[MonitorObject]uint32{
MonitorActionAny: {
MonitorObjectAny: 0xffffffff,
MonitorObjectTables: 1<<unix.NFT_MSG_NEWTABLE | 1<<unix.NFT_MSG_DELTABLE,
MonitorObjectChains: 1<<unix.NFT_MSG_NEWCHAIN | 1<<unix.NFT_MSG_DELCHAIN,
MonitorObjectRules: 1<<unix.NFT_MSG_NEWRULE | 1<<unix.NFT_MSG_DELRULE,
MonitorObjectSets: 1<<unix.NFT_MSG_NEWSET | 1<<unix.NFT_MSG_DELSET,
MonitorObjectElements: 1<<unix.NFT_MSG_NEWSETELEM | 1<<unix.NFT_MSG_DELSETELEM,
MonitorObjectRuleset: 1<<unix.NFT_MSG_NEWTABLE | 1<<unix.NFT_MSG_DELTABLE |
1<<unix.NFT_MSG_NEWCHAIN | 1<<unix.NFT_MSG_DELCHAIN |
1<<unix.NFT_MSG_NEWRULE | 1<<unix.NFT_MSG_DELRULE |
1<<unix.NFT_MSG_NEWSET | 1<<unix.NFT_MSG_DELSET |
1<<unix.NFT_MSG_NEWSETELEM | 1<<unix.NFT_MSG_DELSETELEM |
1<<unix.NFT_MSG_NEWOBJ | 1<<unix.NFT_MSG_DELOBJ,
},
MonitorActionNew: {
MonitorObjectAny: 1<<unix.NFT_MSG_NEWTABLE |
1<<unix.NFT_MSG_NEWCHAIN |
1<<unix.NFT_MSG_NEWRULE |
1<<unix.NFT_MSG_NEWSET |
1<<unix.NFT_MSG_NEWSETELEM,
MonitorObjectTables: 1 << unix.NFT_MSG_NEWTABLE,
MonitorObjectChains: 1 << unix.NFT_MSG_NEWCHAIN,
MonitorObjectRules: 1 << unix.NFT_MSG_NEWRULE,
MonitorObjectSets: 1 << unix.NFT_MSG_NEWSET,
MonitorObjectRuleset: 1<<unix.NFT_MSG_NEWTABLE |
1<<unix.NFT_MSG_NEWCHAIN |
1<<unix.NFT_MSG_NEWRULE |
1<<unix.NFT_MSG_NEWSET |
1<<unix.NFT_MSG_NEWSETELEM |
1<<unix.NFT_MSG_NEWOBJ,
},
MonitorActionDel: {
MonitorObjectAny: 1<<unix.NFT_MSG_DELTABLE |
1<<unix.NFT_MSG_DELCHAIN |
1<<unix.NFT_MSG_DELRULE |
1<<unix.NFT_MSG_DELSET |
1<<unix.NFT_MSG_DELSETELEM |
1<<unix.NFT_MSG_DELOBJ,
},
}
)
type MonitorEventType int
const (
MonitorEventTypeNewTable MonitorEventType = unix.NFT_MSG_NEWTABLE
MonitorEventTypeDelTable MonitorEventType = unix.NFT_MSG_DELTABLE
MonitorEventTypeNewChain MonitorEventType = unix.NFT_MSG_NEWCHAIN
MonitorEventTypeDelChain MonitorEventType = unix.NFT_MSG_DELCHAIN
MonitorEventTypeNewRule MonitorEventType = unix.NFT_MSG_NEWRULE
MonitorEventTypeDelRule MonitorEventType = unix.NFT_MSG_DELRULE
MonitorEventTypeNewSet MonitorEventType = unix.NFT_MSG_NEWSET
MonitorEventTypeDelSet MonitorEventType = unix.NFT_MSG_DELSET
MonitorEventTypeNewSetElem MonitorEventType = unix.NFT_MSG_NEWSETELEM
MonitorEventTypeDelSetElem MonitorEventType = unix.NFT_MSG_DELSETELEM
MonitorEventTypeNewObj MonitorEventType = unix.NFT_MSG_NEWOBJ
MonitorEventTypeDelObj MonitorEventType = unix.NFT_MSG_DELOBJ
MonitorEventTypeOOB MonitorEventType = math.MaxInt // out of band event
)
// A MonitorEvent represents a single change received via a [Monitor].
//
// Depending on the Type, the Data field can be type-asserted to the specific
// data type for this event, e.g. when Type is
// nftables.MonitorEventTypeNewTable, you can access the corresponding table
// details via Data.(*nftables.Table).
type MonitorEvent struct {
Type MonitorEventType
Data any
Error error
}
const (
monitorOK = iota
monitorClosed
)
// A Monitor is an event-based nftables monitor that will receive one event per
// new (or deleted) table, chain, rule, set, etc., depending on the monitor
// configuration.
type Monitor struct {
action MonitorAction
object MonitorObject
monitorFlags uint32
conn *netlink.Conn
closer netlinkCloser
// mu covers eventCh and status
mu sync.Mutex
eventCh chan *MonitorEvent
status int
}
type MonitorOption func(*Monitor)
func WithMonitorEventBuffer(size int) MonitorOption {
return func(monitor *Monitor) {
monitor.eventCh = make(chan *MonitorEvent, size)
}
}
// WithMonitorAction to set monitor actions like new, del or any.
func WithMonitorAction(action MonitorAction) MonitorOption {
return func(monitor *Monitor) {
monitor.action = action
}
}
// WithMonitorObject to set monitor objects.
func WithMonitorObject(object MonitorObject) MonitorOption {
return func(monitor *Monitor) {
monitor.object = object
}
}
// NewMonitor returns a Monitor with options to be started.
//
// Note that NewMonitor only prepares a Monitor. To install the monitor, call
// [Conn.AddMonitor].
func NewMonitor(opts ...MonitorOption) *Monitor {
monitor := &Monitor{
status: monitorOK,
}
for _, opt := range opts {
opt(monitor)
}
if monitor.eventCh == nil {
monitor.eventCh = make(chan *MonitorEvent)
}
objects, ok := monitorFlags[monitor.action]
if !ok {
objects = monitorFlags[MonitorActionAny]
}
flags, ok := objects[monitor.object]
if !ok {
flags = objects[MonitorObjectAny]
}
monitor.monitorFlags = flags
return monitor
}
func (monitor *Monitor) monitor() {
for {
msgs, err := monitor.conn.Receive()
if err != nil {
if strings.Contains(err.Error(), "use of closed file") {
// ignore the error that be closed
break
} else {
// any other errors will be send to user, and then to close eventCh
event := &MonitorEvent{
Type: MonitorEventTypeOOB,
Data: nil,
Error: err,
}
monitor.eventCh <- event
break
}
}
for _, msg := range msgs {
if msg.Header.Type&0xff00>>8 != netlink.HeaderType(unix.NFNL_SUBSYS_NFTABLES) {
continue
}
msgType := msg.Header.Type & 0x00ff
if monitor.monitorFlags&1<<msgType == 0 {
continue
}
switch msgType {
case unix.NFT_MSG_NEWTABLE, unix.NFT_MSG_DELTABLE:
table, err := tableFromMsg(msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: table,
Error: err,
}
monitor.eventCh <- event
case unix.NFT_MSG_NEWCHAIN, unix.NFT_MSG_DELCHAIN:
chain, err := chainFromMsg(msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: chain,
Error: err,
}
monitor.eventCh <- event
case unix.NFT_MSG_NEWRULE, unix.NFT_MSG_DELRULE:
rule, err := parseRuleFromMsg(msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: rule,
Error: err,
}
monitor.eventCh <- event
case unix.NFT_MSG_NEWSET, unix.NFT_MSG_DELSET:
set, err := setsFromMsg(msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: set,
Error: err,
}
monitor.eventCh <- event
case unix.NFT_MSG_NEWSETELEM, unix.NFT_MSG_DELSETELEM:
elems, err := elementsFromMsg(uint8(TableFamilyUnspecified), msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: elems,
Error: err,
}
monitor.eventCh <- event
case unix.NFT_MSG_NEWOBJ, unix.NFT_MSG_DELOBJ:
obj, err := objFromMsg(msg)
event := &MonitorEvent{
Type: MonitorEventType(msgType),
Data: obj,
Error: err,
}
monitor.eventCh <- event
}
}
}
monitor.mu.Lock()
defer monitor.mu.Unlock()
if monitor.status != monitorClosed {
monitor.status = monitorClosed
}
close(monitor.eventCh)
}
func (monitor *Monitor) Close() error {
monitor.mu.Lock()
defer monitor.mu.Unlock()
if monitor.status != monitorClosed {
monitor.status = monitorClosed
return monitor.closer()
}
return nil
}
// AddMonitor to perform the monitor immediately. The channel will be closed after
// calling Close on Monitor or encountering a netlink conn error while Receive.
// Caller may receive a MonitorEventTypeOOB event which contains an error we didn't
// handle, for now.
func (cc *Conn) AddMonitor(monitor *Monitor) (chan *MonitorEvent, error) {
conn, closer, err := cc.netlinkConn()
if err != nil {
return nil, err
}
monitor.conn = conn
monitor.closer = closer
if monitor.monitorFlags != 0 {
if err = conn.JoinGroup(uint32(unix.NFNLGRP_NFTABLES)); err != nil {
monitor.closer()
return nil, err
}
}
go monitor.monitor()
return monitor.eventCh, nil
}
func parseRuleFromMsg(msg netlink.Message) (*Rule, error) {
genmsg := &NFGenMsg{}
genmsg.Decode(msg.Data[:4])
return ruleFromMsg(TableFamily(genmsg.NFGenFamily), msg)
}
+22 -5
View File
@@ -22,7 +22,10 @@ import (
"golang.org/x/sys/unix"
)
var objHeaderType = netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWOBJ)
var (
newObjHeaderType = netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_NEWOBJ)
delObjHeaderType = netlink.HeaderType((unix.NFNL_SUBSYS_NFTABLES << 8) | unix.NFT_MSG_DELOBJ)
)
// Obj represents a netfilter stateful object. See also
// https://wiki.nftables.org/wiki-nftables/index.php/Stateful_objects
@@ -125,8 +128,8 @@ func (cc *Conn) ResetObjects(t *Table) ([]Obj, error) {
}
func objFromMsg(msg netlink.Message) (Obj, error) {
if got, want := msg.Header.Type, objHeaderType; got != want {
return nil, fmt.Errorf("unexpected header type: got %v, want %v", got, want)
if got, want1, want2 := msg.Header.Type, newObjHeaderType, delObjHeaderType; got != want1 && got != want2 {
return nil, fmt.Errorf("unexpected header type: got %v, want %v or %v", got, want1, want2)
}
ad, err := netlink.NewAttributeDecoder(msg.Data[4:])
if err != nil {
@@ -138,7 +141,6 @@ func objFromMsg(msg netlink.Message) (Obj, error) {
name string
objectType uint32
)
const NFT_OBJECT_COUNTER = 1 // TODO: get into x/sys/unix
for ad.Next() {
switch ad.Type() {
case unix.NFTA_OBJ_TABLE:
@@ -149,12 +151,27 @@ func objFromMsg(msg netlink.Message) (Obj, error) {
objectType = ad.Uint32()
case unix.NFTA_OBJ_DATA:
switch objectType {
case NFT_OBJECT_COUNTER:
case unix.NFT_OBJECT_COUNTER:
o := CounterObj{
Table: table,
Name: name,
}
ad.Do(func(b []byte) error {
ad, err := netlink.NewAttributeDecoder(b)
if err != nil {
return err
}
ad.ByteOrder = binary.BigEndian
return o.unmarshal(ad)
})
return &o, ad.Err()
case NFT_OBJECT_QUOTA:
o := QuotaObj{
Table: table,
Name: name,
}
ad.Do(func(b []byte) error {
ad, err := netlink.NewAttributeDecoder(b)
if err != nil {
Loaded 100 of 518 files, more files were not shown because too many files have changed in this diff. Show more