Add sqlite pragmas to db config

Feature/website info

README.md

@@ -43,4 +43,3 @@ See the [wiki](https://sliver.sh/docs?name=Compile+from+Source).
 
 Sliver is licensed under [GPLv3](https://www.gnu.org/licenses/gpl-3.0.en.html), some sub-components may have separate licenses. See their respective subdirectories in this project for details.
 
-

client/command/websites/commands.go

@@ -1,10 +1,14 @@
 package websites
 
 import (
+	"context"
+	"strings"
+
 	"github.com/bishopfox/sliver/client/command/flags"
 	"github.com/bishopfox/sliver/client/command/help"
 	"github.com/bishopfox/sliver/client/console"
 	consts "github.com/bishopfox/sliver/client/constants"
+	"github.com/bishopfox/sliver/protobuf/commonpb"
 	"github.com/rsteube/carapace"
 	"github.com/spf13/cobra"
 	"github.com/spf13/pflag"
@@ -26,6 +30,41 @@ func Commands(con *console.SliverClient) []*cobra.Command {
 	})
 	carapace.Gen(websitesCmd).PositionalCompletion(WebsiteNameCompleter(con))
 
+	websitesListCmd := &cobra.Command{
+		Use:   consts.ListStr,
+		Short: "List configured websites",
+		Long:  help.GetHelpFor([]string{consts.WebsitesStr, consts.ListStr}),
+		Run: func(cmd *cobra.Command, args []string) {
+			ListWebsites(cmd, con, args)
+		},
+	}
+	websitesCmd.AddCommand(websitesListCmd)
+
+	websitesShowCmd := &cobra.Command{
+		Use:   "show [name]",
+		Short: "Show contents of a website",
+		Long:  help.GetHelpFor([]string{consts.WebsitesStr}),
+		Args:  cobra.ExactArgs(1),
+		Run: func(cmd *cobra.Command, args []string) {
+			ListWebsiteContent(args[0], con)
+		},
+	}
+	carapace.Gen(websitesShowCmd).PositionalCompletion(WebsiteNameCompleter(con))
+	websitesShowCmd.ValidArgsFunction = func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) {
+		ws, err := con.Rpc.Websites(context.Background(), &commonpb.Empty{})
+		if err != nil {
+			return nil, cobra.ShellCompDirectiveError
+		}
+		out := []string{}
+		for _, w := range ws.Websites {
+			if strings.HasPrefix(w.Name, toComplete) {
+				out = append(out, w.Name)
+			}
+		}
+		return out, cobra.ShellCompDirectiveNoFileComp
+	}
+	websitesCmd.AddCommand(websitesShowCmd)
+
 	websitesRmCmd := &cobra.Command{
 		Use:   consts.RmStr,
 		Short: "Remove an entire website and all of its contents",

client/command/websites/websites-add-content.go

@@ -121,9 +121,10 @@ func webAddFile(web *clientpb.WebsiteAddContent, webpath string, contentType str
 	}
 
 	web.Contents[webpath] = &clientpb.WebContent{
-		Path:        webpath,
-		ContentType: contentType,
-		Content:     data,
+		Path:         webpath,
+		ContentType:  contentType,
+		OriginalFile: filepath.Base(contentPath),
+		Content:      data,
 	}
 	return nil
 }

client/command/websites/websites.go

@@ -21,7 +21,6 @@ package websites
 import (
 	"context"
 	"sort"
-	"strings"
 
 	"github.com/bishopfox/sliver/client/command/settings"
 	"github.com/bishopfox/sliver/client/console"
@@ -58,11 +57,14 @@ func ListWebsites(cmd *cobra.Command, con *console.SliverClient, args []string)
 		con.PrintInfof("No websites\n")
 		return
 	}
-	con.Println("Websites")
-	con.Println(strings.Repeat("=", len("Websites")))
+	tw := table.NewWriter()
+	tw.SetStyle(settings.GetTableStyle(con))
+	tw.AppendHeader(table.Row{"Name", "Objects"})
 	for _, site := range websites.Websites {
-		con.Printf("%s%s%s - %d page(s)\n", console.Bold, site.Name, console.Normal, len(site.Contents))
+		tw.AppendRow(table.Row{site.Name, len(site.Contents)})
 	}
+
+	con.Println(tw.Render())
 }
 
 // ListWebsiteContent - List the static contents of a website.
@@ -91,6 +93,8 @@ func PrintWebsite(web *clientpb.Website, con *console.SliverClient) {
 		"Path",
 		"Content-type",
 		"Size",
+		"Original File",
+		"SHA256",
 	})
 	sortedContents := []*clientpb.WebContent{}
 	for _, content := range web.Contents {
@@ -104,6 +108,8 @@ func PrintWebsite(web *clientpb.Website, con *console.SliverClient) {
 			content.Path,
 			content.ContentType,
 			content.Size,
+			content.OriginalFile,
+			content.Sha256,
 		})
 	}
 	con.Println(tw.Render())

protobuf/clientpb/client.proto

@@ -586,6 +586,9 @@ message WebContent {
   string ContentType = 4;
   uint64 Size = 5 [ jstype = JS_STRING ];
 
+  string OriginalFile = 6;
+  string Sha256 = 7;
+
   bytes Content = 9;
 }
 
@@ -1316,4 +1319,4 @@ message CertificateData {
 
 message CertificateInfo {
   repeated CertificateData info = 1;  
-}
+}

protobuf/commonpb/common.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v6.33.2
+// 	protoc        v3.15.8
 // source: commonpb/common.proto
 
 package commonpb

protobuf/dnspb/dns.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v6.33.2
+// 	protoc        v3.15.8
 // source: dnspb/dns.proto
 
 package dnspb

protobuf/rpcpb/services.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v6.33.2
+// 	protoc        v3.15.8
 // source: rpcpb/services.proto
 
 package rpcpb

protobuf/sliverpb/sliver.pb.go

@@ -1,7 +1,7 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
 // versions:
 // 	protoc-gen-go v1.26.0
-// 	protoc        v6.33.2
+// 	protoc        v3.15.8
 // source: sliverpb/sliver.proto
 
 package sliverpb

server/configs/database.go

@@ -26,6 +26,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"sort"
 
 	"github.com/bishopfox/sliver/server/assets"
 	"github.com/bishopfox/sliver/server/log"
@@ -47,6 +48,13 @@ var (
 	ErrInvalidDialect = errors.New("invalid SQL Dialect")
 
 	databaseConfigLog = log.NamedLogger("config", "database")
+
+	defaultSQLitePragmas = map[string]string{
+		"journal_mode": "WAL",    // reduce writer blocking, better concurrency
+		"busy_timeout": "5000",   // wait for locks instead of failing fast (ms)
+		"synchronous":  "NORMAL", // faster WAL syncs while retaining durability
+		"temp_store":   "MEMORY", // keep temp structures off disk for quicker startup
+	}
 )
 
 // GetDatabaseConfigPath - File path to config.json
@@ -66,7 +74,8 @@ type DatabaseConfig struct {
 	Host     string `json:"host"`
 	Port     uint16 `json:"port"`
 
-	Params map[string]string `json:"params"`
+	Params  map[string]string `json:"params"`
+	Pragmas map[string]string `json:"pragmas"`
 
 	MaxIdleConns int `json:"max_idle_conns"`
 	MaxOpenConns int `json:"max_open_conns"`
@@ -80,7 +89,7 @@ func (c *DatabaseConfig) DSN() (string, error) {
 	switch c.Dialect {
 	case Sqlite:
 		filePath := filepath.Join(assets.GetRootAppDir(), "sliver.db")
-		params := encodeParams(c.Params)
+		params := encodeSQLiteParams(c.Params, c.Pragmas)
 		return fmt.Sprintf("file:%s?%s", filePath, params), nil
 	case MySQL:
 		user := url.QueryEscape(c.Username)
@@ -112,6 +121,44 @@ func encodeParams(rawParams map[string]string) string {
 	return params.Encode()
 }
 
+func encodeSQLiteParams(rawParams map[string]string, pragmas map[string]string) string {
+	if rawParams == nil {
+		rawParams = map[string]string{}
+	}
+	params := url.Values{}
+
+	// Preserve any user-provided parameters first.
+	for key, value := range rawParams {
+		params.Add(key, value)
+	}
+
+	// Apply safer defaults only when the user has not set any custom pragma.
+	if _, ok := rawParams["_pragma"]; !ok {
+		selectedPragmas := pragmas
+		if len(selectedPragmas) == 0 {
+			selectedPragmas = defaultSQLitePragmas
+		}
+
+		keys := make([]string, 0, len(selectedPragmas))
+		for key := range selectedPragmas {
+			keys = append(keys, key)
+		}
+		sort.Strings(keys)
+
+		for _, key := range keys {
+			value := selectedPragmas[key]
+			params.Add("_pragma", fmt.Sprintf("%s(%s)", key, value))
+		}
+	}
+
+	// Encourage shared cache to play nicer with WAL and multiple connections.
+	if _, ok := rawParams["cache"]; !ok {
+		params.Add("cache", "shared")
+	}
+
+	return params.Encode()
+}
+
 // Save - Save config file to disk
 func (c *DatabaseConfig) Save() error {
 	configPath := GetDatabaseConfigPath()
@@ -161,6 +208,8 @@ func GetDatabaseConfig() *DatabaseConfig {
 		config.MaxOpenConns = 1
 	}
 
+	ensureSQLiteDefaults(config)
+
 	err := config.Save() // This updates the config with any missing fields
 	if err != nil {
 		databaseConfigLog.Errorf("Failed to save default config %s", err)
@@ -171,9 +220,29 @@ func GetDatabaseConfig() *DatabaseConfig {
 func getDefaultDatabaseConfig() *DatabaseConfig {
 	return &DatabaseConfig{
 		Dialect:      Sqlite,
+		Pragmas:      defaultSQLitePragmas,
 		MaxIdleConns: 10,
 		MaxOpenConns: 100,
 
 		LogLevel: "warn",
 	}
 }
+
+func ensureSQLiteDefaults(c *DatabaseConfig) {
+	if c.Dialect != Sqlite {
+		return
+	}
+	if c.Params == nil {
+		c.Params = map[string]string{}
+	}
+	if _, ok := c.Params["_pragma"]; ok {
+		// User provided explicit pragmas; leave Pragmas untouched to avoid confusion.
+		return
+	}
+	if len(c.Pragmas) == 0 {
+		c.Pragmas = defaultSQLitePragmas
+	}
+	if _, ok := c.Params["cache"]; !ok {
+		c.Params["cache"] = "shared"
+	}
+}

server/db/helpers.go

@@ -789,8 +789,18 @@ func AddContent(pbWebContent *clientpb.WebContent, webContentDir string) (*clien
 			return nil, err
 		}
 	} else {
-		dbWebContent.ContentType = pbWebContent.ContentType
-		dbWebContent.Size = pbWebContent.Size
+		if pbWebContent.ContentType != "" {
+			dbWebContent.ContentType = pbWebContent.ContentType
+		}
+		if pbWebContent.Size != 0 {
+			dbWebContent.Size = pbWebContent.Size
+		}
+		if pbWebContent.OriginalFile != "" {
+			dbWebContent.OriginalFile = pbWebContent.OriginalFile
+		}
+		if pbWebContent.Sha256 != "" {
+			dbWebContent.Sha256 = pbWebContent.Sha256
+		}
 
 		dbModelWebContent := models.WebContentFromProtobuf(dbWebContent)
 		err = Session().Save(&dbModelWebContent).Error

server/db/models/website.go

@@ -67,9 +67,11 @@ type WebContent struct {
 	ID        uuid.UUID `gorm:"primaryKey;->;<-:create;type:uuid;"`
 	WebsiteID uuid.UUID `gorm:"type:uuid;"`
 
-	Path        string `gorm:"primaryKey"`
-	Size        uint64
-	ContentType string
+	Path         string `gorm:"primaryKey"`
+	Size         uint64
+	ContentType  string
+	OriginalFile string
+	Sha256       string
 }
 
 // BeforeCreate - GORM hook to automatically set values
@@ -81,12 +83,14 @@ func (wc *WebContent) BeforeCreate(tx *gorm.DB) (err error) {
 // ToProtobuf - Converts to protobuf object
 func (wc *WebContent) ToProtobuf(content *[]byte) *clientpb.WebContent {
 	return &clientpb.WebContent{
-		ID:          wc.ID.String(),
-		WebsiteID:   wc.WebsiteID.String(),
-		Path:        wc.Path,
-		Size:        uint64(wc.Size),
-		ContentType: wc.ContentType,
-		Content:     *content,
+		ID:           wc.ID.String(),
+		WebsiteID:    wc.WebsiteID.String(),
+		Path:         wc.Path,
+		Size:         uint64(wc.Size),
+		ContentType:  wc.ContentType,
+		OriginalFile: wc.OriginalFile,
+		Sha256:       wc.Sha256,
+		Content:      *content,
 	}
 }
 
@@ -95,10 +99,12 @@ func WebContentFromProtobuf(pbWebContent *clientpb.WebContent) WebContent {
 	websiteUUID, _ := uuid.FromString(pbWebContent.WebsiteID)
 
 	return WebContent{
-		ID:          siteUUID,
-		WebsiteID:   websiteUUID,
-		Path:        pbWebContent.Path,
-		Size:        pbWebContent.Size,
-		ContentType: pbWebContent.ContentType,
+		ID:           siteUUID,
+		WebsiteID:    websiteUUID,
+		Path:         pbWebContent.Path,
+		Size:         pbWebContent.Size,
+		ContentType:  pbWebContent.ContentType,
+		OriginalFile: pbWebContent.OriginalFile,
+		Sha256:       pbWebContent.Sha256,
 	}
 }

server/website/website.go

@@ -19,6 +19,8 @@ package website
 */
 
 import (
+	"crypto/sha256"
+	"encoding/hex"
 	"net/url"
 	"os"
 	"path/filepath"
@@ -87,14 +89,30 @@ func AddContent(name string, pbWebContent *clientpb.WebContent) error {
 		pbWebContent.WebsiteID = website.ID
 	}
 
+	if pbWebContent.Size == 0 && len(pbWebContent.Content) > 0 {
+		pbWebContent.Size = uint64(len(pbWebContent.Content))
+	}
+	if pbWebContent.OriginalFile == "" {
+		pbWebContent.OriginalFile = filepath.Base(pbWebContent.Path)
+	}
+
+	if len(pbWebContent.Content) > 0 {
+		sha := sha256.Sum256(pbWebContent.Content)
+		pbWebContent.Sha256 = hex.EncodeToString(sha[:])
+	}
+
 	webContent, err := db.AddContent(pbWebContent, webContentDir)
 	if err != nil {
 		return err
 	}
 
-	// Write content to disk
-	webContentPath := filepath.Join(webContentDir, webContent.ID)
-	return os.WriteFile(webContentPath, pbWebContent.Content, 0600)
+	// Write content to disk when provided (metadata-only updates skip disk writes)
+	if len(pbWebContent.Content) > 0 {
+		webContentPath := filepath.Join(webContentDir, webContent.ID)
+		return os.WriteFile(webContentPath, pbWebContent.Content, 0600)
+	}
+
+	return nil
 }
 
 // RemoveContent - Remove website content for a path