Merge pull request #7658 from lucasonline0/fix/7651-ignore-file-validation

fix(installer): validate nuclei ignore file downloads

pkg/catalog/config/ignorefile.go

@@ -1,15 +1,25 @@
 package config
 
 import (
+	"bytes"
 	"crypto/md5"
 	"errors"
 	"fmt"
 	"io"
 	"os"
+	"reflect"
+	"slices"
+	"strings"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/utils/yaml"
 )
 
+// maxIgnoreFileBytes caps untrusted ignore-file payloads (PDTM downloads,
+// template archives) before YAML decode.
+const maxIgnoreFileBytes = 1 << 20
+
+var errMultipleIgnoreFileDocuments = errors.New("multiple YAML documents are not supported")
+
 // IgnoreFile is an internal nuclei template blocking configuration file
 type IgnoreFile struct {
 	Tags  []string `yaml:"tags"`
@@ -30,33 +40,102 @@ func ReadIgnoreFile() (IgnoreFile, error) {
 		_ = file.Close()
 	}()
 
-	decoder := yaml.NewDecoder(file)
-	ignore := IgnoreFile{}
-	if err := decoder.Decode(&ignore); err != nil {
+	ignore, err := decodeIgnoreFile(file, false)
+	if err != nil {
 		return IgnoreFile{}, fmt.Errorf("error parsing %q: %w", path, err)
 	}
 
-	var additionalDocument any
-	if err := decoder.Decode(&additionalDocument); !errors.Is(err, io.EOF) {
-		if err != nil {
-			return IgnoreFile{}, fmt.Errorf("error parsing %q: %w", path, err)
-		}
-
-		return IgnoreFile{}, fmt.Errorf("error parsing %q: multiple YAML documents are not supported", path)
-	}
-
 	return ignore, nil
 }
 
 // WriteActiveIgnoreFile atomically replaces the active root ignore file.
 func (c *Config) WriteActiveIgnoreFile(data []byte) error {
-	if err := atomicWriteFile(c.GetActiveIgnoreFilePath(), data, 0o600); err != nil {
+	path := c.GetActiveIgnoreFilePath()
+	if err := validateIgnoreFileContents(data); err != nil {
+		return fmt.Errorf("invalid active .nuclei-ignore file %q: %w", path, err)
+	}
+
+	if err := atomicWriteFile(path, data, 0o600); err != nil {
 		return fmt.Errorf("write active .nuclei-ignore file: %w", err)
 	}
 
 	return nil
 }
 
+func validateIgnoreFileContents(data []byte) error {
+	if len(data) > maxIgnoreFileBytes {
+		return fmt.Errorf("payload exceeds %d bytes", maxIgnoreFileBytes)
+	}
+	if len(bytes.TrimSpace(data)) == 0 {
+		return errors.New("empty document")
+	}
+
+	if _, err := decodeIgnoreFile(bytes.NewReader(data), true); err != nil {
+		if errors.Is(err, io.EOF) {
+			return errors.New("empty document")
+		}
+		return err
+	}
+
+	decoder := yaml.NewDecoder(bytes.NewReader(data))
+	var raw map[string]any
+	if err := decoder.Decode(&raw); err != nil {
+		return err
+	}
+	if raw == nil {
+		return errors.New("document must be a YAML mapping")
+	}
+
+	allowed := ignoreFileFieldNames()
+	found := false
+	for key := range raw {
+		if !slices.Contains(allowed, key) {
+			return fmt.Errorf("unknown field %q", key)
+		}
+		found = true
+	}
+	if !found {
+		return fmt.Errorf("document must define at least one of %s", strings.Join(allowed, ", "))
+	}
+
+	return nil
+}
+
+func decodeIgnoreFile(r io.Reader, knownFields bool) (IgnoreFile, error) {
+	decoder := yaml.NewDecoder(r)
+	decoder.KnownFields(knownFields)
+
+	ignore := IgnoreFile{}
+	if err := decoder.Decode(&ignore); err != nil {
+		return IgnoreFile{}, err
+	}
+
+	var additionalDocument any
+	if err := decoder.Decode(&additionalDocument); !errors.Is(err, io.EOF) {
+		if err != nil {
+			return IgnoreFile{}, err
+		}
+
+		return IgnoreFile{}, errMultipleIgnoreFileDocuments
+	}
+
+	return ignore, nil
+}
+
+func ignoreFileFieldNames() []string {
+	typ := reflect.TypeOf(IgnoreFile{})
+	names := make([]string, 0, typ.NumField())
+	for i := 0; i < typ.NumField(); i++ {
+		name, _, _ := strings.Cut(typ.Field(i).Tag.Get("yaml"), ",")
+		if name == "" || name == "-" {
+			continue
+		}
+		names = append(names, name)
+	}
+	slices.Sort(names)
+	return names
+}
+
 // IgnoreFileNeedsUpdate reports whether the active ignore file is
 // missing or differs from the latest known hash.
 func (c *Config) IgnoreFileNeedsUpdate(latestHash string) bool {

pkg/catalog/config/ignorefile_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"os"
 	"path/filepath"
@@ -108,6 +109,77 @@ func TestReadIgnoreFileParsesActiveFile(t *testing.T) {
 	}
 }
 
+func TestWriteActiveIgnoreFileRejectsInvalidPayloads(t *testing.T) {
+	root := setActiveIgnoreFileRoot(t)
+	cfg := DefaultConfig
+	path := filepath.Join(root, NucleiIgnoreFileName)
+	valid := []byte("tags: [weak]\nfiles: [blocked.yaml]\n")
+	if err := os.WriteFile(path, valid, 0o600); err != nil {
+		t.Fatalf("seed ignore file: %v", err)
+	}
+
+	cases := map[string][]byte{
+		"proxy html":          []byte("<!DOCTYPE HTML>\n<html>\n<head>\n<meta name=\"description\" content=\"Zscaler: blocked\">\n</head>\n</html>\n"),
+		"unrelated yaml":      []byte("error: access denied\nmessage: blocked by proxy\n"),
+		"empty":               []byte("   \n"),
+		"empty mapping":       []byte("{}\n"),
+		"null document":       []byte("null\n"),
+		"template yaml":       []byte("id: example\ninfo:\n  name: example\n"),
+		"additional document": []byte("tags: [weak]\n---\nfiles: [blocked.yaml]\n"),
+		"malformed yaml":      []byte("tags: ["),
+		"oversized":           bytes.Repeat([]byte("a"), maxIgnoreFileBytes+1),
+	}
+
+	for name, payload := range cases {
+		t.Run(name, func(t *testing.T) {
+			err := cfg.WriteActiveIgnoreFile(payload)
+			if err == nil {
+				t.Fatal("write invalid ignore file succeeded")
+			}
+			if message := err.Error(); !strings.Contains(message, strconv.Quote(path)) {
+				t.Fatalf("invalid write error = %q, want active path", message)
+			}
+			got, readErr := os.ReadFile(path)
+			if readErr != nil {
+				t.Fatalf("read ignore file: %v", readErr)
+			}
+			if !bytes.Equal(got, valid) {
+				t.Fatalf("invalid payload overwrote ignore file: got %q", got)
+			}
+		})
+	}
+}
+
+func TestWriteActiveIgnoreFileAcceptsKnownShapes(t *testing.T) {
+	root := setActiveIgnoreFileRoot(t)
+	cfg := DefaultConfig
+	path := filepath.Join(root, NucleiIgnoreFileName)
+
+	cases := map[string][]byte{
+		"tags and files": []byte("tags: [weak]\nfiles: [blocked.yaml]\n"),
+		"tags only":      []byte("tags: []\n"),
+		"files only":     []byte("files: [blocked.yaml]\n"),
+	}
+
+	for name, payload := range cases {
+		t.Run(name, func(t *testing.T) {
+			if err := cfg.WriteActiveIgnoreFile(payload); err != nil {
+				t.Fatalf("write ignore file: %v", err)
+			}
+			got, err := os.ReadFile(path)
+			if err != nil {
+				t.Fatalf("read ignore file: %v", err)
+			}
+			if !bytes.Equal(got, payload) {
+				t.Fatalf("ignore file = %q, want %q", got, payload)
+			}
+			if _, err := ReadIgnoreFile(); err != nil {
+				t.Fatalf("read back written ignore file: %v", err)
+			}
+		})
+	}
+}
+
 func TestSetTemplatesDirDoesNotCopyLegacyIgnoreFile(t *testing.T) {
 	dir := t.TempDir()
 	configDir := filepath.Join(dir, "config")

pkg/installer/template.go

@@ -492,6 +492,10 @@ func (t *TemplateManager) getTemplateOutputLocation(templateDir, uri string, f f
 	return templateDir, newPath
 }
 
+func isActiveIgnoreFilePath(writePath string) bool {
+	return filepath.Clean(writePath) == filepath.Clean(config.DefaultConfig.GetActiveIgnoreFilePath())
+}
+
 // writeTemplatesToDisk writes release outputs to disk and returns their digests.
 // The returned map includes every successfully written output; ownership
 // reconciliation filters it to official template paths.
@@ -595,6 +599,14 @@ type templateOutputWriteResult struct {
 
 func writeTemplateOutput(rootDir, writePath string, contents []byte, mode fs.FileMode) (templateOutputWriteResult, error) {
 	var result templateOutputWriteResult
+	if isActiveIgnoreFilePath(writePath) {
+		if err := config.DefaultConfig.WriteActiveIgnoreFile(contents); err != nil {
+			return result, err
+		}
+		result.touchedDirectories = append(result.touchedDirectories, filepath.Dir(writePath))
+		return result, nil
+	}
+
 	relativePath, err := filepath.Rel(rootDir, writePath)
 	if err != nil {
 		return result, err

pkg/installer/template_test.go

@@ -278,6 +278,33 @@ func TestGetTemplateOutputLocationRoutesIgnoreFile(t *testing.T) {
 	require.Equal(t, cfg.GetActiveIgnoreFilePath(), writePath)
 }
 
+func TestWriteTemplateOutputValidatesIgnoreFile(t *testing.T) {
+	previousConfig := config.DefaultConfig
+	templatesDir := t.TempDir()
+	cfg := &config.Config{}
+	cfg.SetTemplatesDir(templatesDir)
+	config.DefaultConfig = cfg
+	t.Cleanup(func() { config.DefaultConfig = previousConfig })
+
+	path := cfg.GetActiveIgnoreFilePath()
+	valid := []byte("tags: [weak]\n")
+	require.NoError(t, os.WriteFile(path, valid, 0o600))
+
+	_, err := writeTemplateOutput(templatesDir, path, []byte("<html>blocked</html>\n"), 0o644)
+	require.Error(t, err)
+
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, valid, got)
+
+	payload := []byte("files: [blocked.yaml]\n")
+	_, err = writeTemplateOutput(templatesDir, path, payload, 0o644)
+	require.NoError(t, err)
+	got, err = os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, payload, got)
+}
+
 func TestCommitTemplateVersionRestoresMemoryOnWriteFailure(t *testing.T) {
 	stateDir := filepath.Join(t.TempDir(), "state")
 	require.NoError(t, os.WriteFile(stateDir, nil, 0o600))

pkg/installer/versioncheck_test.go

@@ -1,13 +1,25 @@
 package installer
 
 import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
 	"testing"
 
 	"github.com/projectdiscovery/nuclei/v3/pkg/catalog/config"
+	"github.com/projectdiscovery/retryablehttp-go"
 	"github.com/projectdiscovery/utils/generic"
 	"github.com/stretchr/testify/require"
 )
 
+type roundTripFunc func(*http.Request) (*http.Response, error)
+
+func (f roundTripFunc) RoundTrip(req *http.Request) (*http.Response, error) {
+	return f(req)
+}
+
 func TestVersionCheck(t *testing.T) {
 	ignoreHash, err := NucleiVersionCheck()
 	require.Nil(t, err)
@@ -17,3 +29,86 @@ func TestVersionCheck(t *testing.T) {
 		t.Errorf("something went wrong got empty response nuclei-version=%v templates-version=%v ignore-hash=%v", cfg.LatestNucleiVersion, cfg.LatestNucleiTemplatesVersion, ignoreHash)
 	}
 }
+
+func TestUpdateIgnoreFileRejectsHTMLAndPreservesExisting(t *testing.T) {
+	html := []byte("<!DOCTYPE HTML>\n<html>\n<head>\n<meta name=\"description\" content=\"Zscaler: blocked\">\n</head>\n</html>\n")
+	withIgnoreDownload(t, http.StatusOK, html)
+
+	cfg := config.DefaultConfig
+	root := t.TempDir()
+	previous := cfg.TemplatesDirectory
+	t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
+	cfg.SetTemplatesDir(root)
+
+	valid := []byte("tags: [weak]\nfiles: [blocked.yaml]\n")
+	path := cfg.GetActiveIgnoreFilePath()
+	require.NoError(t, os.WriteFile(path, valid, 0o600))
+
+	err := UpdateIgnoreFile()
+	require.Error(t, err)
+	require.Contains(t, err.Error(), fmt.Sprintf("%q", path))
+
+	got, err := os.ReadFile(path)
+	require.NoError(t, err)
+	require.Equal(t, valid, got)
+}
+
+func TestUpdateIgnoreFileWritesValidContents(t *testing.T) {
+	payload := []byte("tags: [dos]\nfiles: [http/test.yaml]\n")
+	withIgnoreDownload(t, http.StatusOK, payload)
+
+	cfg := config.DefaultConfig
+	root := t.TempDir()
+	previous := cfg.TemplatesDirectory
+	t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
+	cfg.SetTemplatesDir(root)
+
+	require.NoError(t, UpdateIgnoreFile())
+
+	got, err := os.ReadFile(cfg.GetActiveIgnoreFilePath())
+	require.NoError(t, err)
+	require.Equal(t, payload, got)
+}
+
+func TestUpdateIgnoreFileRejectsUnexpectedStatus(t *testing.T) {
+	withIgnoreDownload(t, http.StatusForbidden, []byte("tags: [weak]\n"))
+
+	cfg := config.DefaultConfig
+	root := t.TempDir()
+	previous := cfg.TemplatesDirectory
+	t.Cleanup(func() { cfg.SetTemplatesDir(previous) })
+	cfg.SetTemplatesDir(root)
+
+	err := UpdateIgnoreFile()
+	require.Error(t, err)
+	require.Contains(t, err.Error(), "unexpected HTTP status")
+	_, statErr := os.Stat(cfg.GetActiveIgnoreFilePath())
+	require.ErrorIs(t, statErr, os.ErrNotExist)
+}
+
+func withIgnoreDownload(t *testing.T, status int, body []byte) {
+	t.Helper()
+
+	previous := retryableHttpClient
+	t.Cleanup(func() { retryableHttpClient = previous })
+
+	retryableHttpClient = retryablehttp.NewClient(retryablehttp.Options{
+		RetryMax: 0,
+		HttpClient: &http.Client{
+			Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) {
+				header := make(http.Header)
+				header.Set("Content-Type", "text/plain")
+				return &http.Response{
+					StatusCode: status,
+					Status:     http.StatusText(status),
+					Body:       io.NopCloser(strings.NewReader(string(body))),
+					Header:     header,
+					Request:    req,
+					Proto:      "HTTP/1.1",
+					ProtoMajor: 1,
+					ProtoMinor: 1,
+				}, nil
+			}),
+		},
+	})
+}