refact: pkg/database decisions filter, queries (#3635)

* refact pkg/database: rename query methods

* refact pkg/database: extract decisionfilter.go

* refact pkg/database: rename filters -> filter

* handle error with since==nil

* lint

pkg/database/alertfilter.go

@@ -255,7 +255,7 @@ func alertPredicatesFromFilter(filter map[string][]string) ([]predicate.Alert, e
 	return predicates, nil
 }
 
-func BuildAlertRequestFromFilter(alerts *ent.AlertQuery, filter map[string][]string) (*ent.AlertQuery, error) {
+func applyAlertFilter(alerts *ent.AlertQuery, filter map[string][]string) (*ent.AlertQuery, error) {
 	preds, err := alertPredicatesFromFilter(filter)
 	if err != nil {
 		return nil, err

pkg/database/alerts.go

@@ -742,7 +742,7 @@ func (c *Client) CreateAlert(ctx context.Context, machineID string, alertList []
 	return alertIDs, nil
 }
 
-func (c *Client) AlertsCountPerScenario(ctx context.Context, filters map[string][]string) (map[string]int, error) {
+func (c *Client) AlertsCountPerScenario(ctx context.Context, filter map[string][]string) (map[string]int, error) {
 	var res []struct {
 		Scenario string
 		Count    int
@@ -750,7 +750,7 @@ func (c *Client) AlertsCountPerScenario(ctx context.Context, filters map[string]
 
 	query := c.Ent.Alert.Query()
 
-	query, err := BuildAlertRequestFromFilter(query, filters)
+	query, err := applyAlertFilter(query, filter)
 	if err != nil {
 		return nil, fmt.Errorf("failed to build alert request: %w", err)
 	}
@@ -801,7 +801,7 @@ func (c *Client) QueryAlertWithFilter(ctx context.Context, filter map[string][]s
 	for {
 		alerts := c.Ent.Alert.Query()
 
-		alerts, err := BuildAlertRequestFromFilter(alerts, filter)
+		alerts, err := applyAlertFilter(alerts, filter)
 		if err != nil {
 			return nil, err
 		}

pkg/database/decisionfilter.go

@@ -0,0 +1,208 @@
+package database
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"github.com/pkg/errors"
+
+	"github.com/crowdsecurity/crowdsec/pkg/csnet"
+	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
+	"github.com/crowdsecurity/crowdsec/pkg/database/ent/decision"
+	"github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate"
+	"github.com/crowdsecurity/crowdsec/pkg/types"
+)
+
+func applyDecisionFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) {
+	var (
+		rng csnet.Range
+		err error
+	)
+
+	contains := true
+	/*if contains is true, return bans that *contains* the given value (value is the inner)
+	  else, return bans that are *contained* by the given value (value is the outer)*/
+
+	/*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */
+	if v, ok := filter["simulated"]; ok {
+		if v[0] == "false" {
+			query = query.Where(decision.SimulatedEQ(false))
+		}
+
+		delete(filter, "simulated")
+	} else {
+		query = query.Where(decision.SimulatedEQ(false))
+	}
+
+	for param, value := range filter {
+		switch param {
+		case "contains":
+			contains, err = strconv.ParseBool(value[0])
+			if err != nil {
+				return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
+			}
+		case "scopes", "scope": // Swagger mentions both of them, let's just support both to make sure we don't break anything
+			scopes := strings.Split(value[0], ",")
+			for i, scope := range scopes {
+				switch strings.ToLower(scope) {
+				case "ip":
+					scopes[i] = types.Ip
+				case "range":
+					scopes[i] = types.Range
+				case "country":
+					scopes[i] = types.Country
+				case "as":
+					scopes[i] = types.AS
+				}
+			}
+
+			query = query.Where(decision.ScopeIn(scopes...))
+		case "value":
+			query = query.Where(decision.ValueEQ(value[0]))
+		case "type":
+			query = query.Where(decision.TypeEQ(value[0]))
+		case "origins":
+			query = query.Where(
+				decision.OriginIn(strings.Split(value[0], ",")...),
+			)
+		case "scenarios_containing":
+			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
+			query = query.Where(decision.Or(predicates...))
+		case "scenarios_not_containing":
+			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
+			query = query.Where(decision.Not(
+				decision.Or(
+					predicates...,
+				),
+			))
+		case "ip", "range":
+			rng, err = csnet.NewRange(value[0])
+			if err != nil {
+				return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
+			}
+		case "limit":
+			limit, err := strconv.Atoi(value[0])
+			if err != nil {
+				return nil, errors.Wrapf(InvalidFilter, "invalid limit value : %s", err)
+			}
+
+			query = query.Limit(limit)
+		case "offset":
+			offset, err := strconv.Atoi(value[0])
+			if err != nil {
+				return nil, errors.Wrapf(InvalidFilter, "invalid offset value : %s", err)
+			}
+
+			query = query.Offset(offset)
+		case "id_gt":
+			id, err := strconv.Atoi(value[0])
+			if err != nil {
+				return nil, errors.Wrapf(InvalidFilter, "invalid id_gt value : %s", err)
+			}
+
+			query = query.Where(decision.IDGT(id))
+		}
+	}
+
+	query, err = decisionIPFilter(query, contains, rng)
+	if err != nil {
+		return nil, fmt.Errorf("fail to apply StartIpEndIpFilter: %w", err)
+	}
+
+	return query, nil
+}
+
+func decisionIPv4Filter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
+	if contains {
+		/*Decision contains {start_ip,end_ip}*/
+		return decisions.Where(decision.And(
+			decision.StartIPLTE(rng.Start.Addr),
+			decision.EndIPGTE(rng.End.Addr),
+			decision.IPSizeEQ(int64(rng.Size())))), nil
+	}
+
+	/*Decision is contained within {start_ip,end_ip}*/
+	return decisions.Where(decision.And(
+		decision.StartIPGTE(rng.Start.Addr),
+		decision.EndIPLTE(rng.End.Addr),
+		decision.IPSizeEQ(int64(rng.Size())))), nil
+}
+
+func decisionIPv6Filter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
+	/*decision contains {start_ip,end_ip}*/
+	if contains {
+		return decisions.Where(decision.And(
+			// matching addr size
+			decision.IPSizeEQ(int64(rng.Size())),
+			decision.Or(
+				// decision.start_ip < query.start_ip
+				decision.StartIPLT(rng.Start.Addr),
+				decision.And(
+					// decision.start_ip == query.start_ip
+					decision.StartIPEQ(rng.Start.Addr),
+					// decision.start_suffix <= query.start_suffix
+					decision.StartSuffixLTE(rng.Start.Sfx),
+				)),
+			decision.Or(
+				// decision.end_ip > query.end_ip
+				decision.EndIPGT(rng.End.Addr),
+				decision.And(
+					// decision.end_ip == query.end_ip
+					decision.EndIPEQ(rng.End.Addr),
+					// decision.end_suffix >= query.end_suffix
+					decision.EndSuffixGTE(rng.End.Sfx),
+				),
+			),
+		)), nil
+	}
+
+	/*decision is contained within {start_ip,end_ip}*/
+	return decisions.Where(decision.And(
+		// matching addr size
+		decision.IPSizeEQ(int64(rng.Size())),
+		decision.Or(
+			// decision.start_ip > query.start_ip
+			decision.StartIPGT(rng.Start.Addr),
+			decision.And(
+				// decision.start_ip == query.start_ip
+				decision.StartIPEQ(rng.Start.Addr),
+				// decision.start_suffix >= query.start_suffix
+				decision.StartSuffixGTE(rng.Start.Sfx),
+			)),
+		decision.Or(
+			// decision.end_ip < query.end_ip
+			decision.EndIPLT(rng.End.Addr),
+			decision.And(
+				// decision.end_ip == query.end_ip
+				decision.EndIPEQ(rng.End.Addr),
+				// decision.end_suffix <= query.end_suffix
+				decision.EndSuffixLTE(rng.End.Sfx),
+			),
+		),
+	)), nil
+}
+
+func decisionIPFilter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
+	switch rng.Size() {
+	case 4:
+		return decisionIPv4Filter(decisions, contains, rng)
+	case 16:
+		return decisionIPv6Filter(decisions, contains, rng)
+	case 0:
+		return decisions, nil
+	default:
+		return nil, errors.Wrapf(InvalidFilter, "unknown ip size %d", rng.Size())
+	}
+}
+
+func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision {
+	words := strings.Split(s, ",")
+	predicates := make([]predicate.Decision, len(words))
+
+	for i, word := range words {
+		predicates[i] = predicateFunc(word)
+	}
+
+	return predicates
+}

pkg/database/decisions.go

@@ -4,7 +4,6 @@ import (
 	"context"
 	"fmt"
 	"strconv"
-	"strings"
 	"time"
 
 	"entgo.io/ent/dialect/sql"
@@ -15,8 +14,6 @@ import (
 	"github.com/crowdsecurity/crowdsec/pkg/csnet"
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent"
 	"github.com/crowdsecurity/crowdsec/pkg/database/ent/decision"
-	"github.com/crowdsecurity/crowdsec/pkg/database/ent/predicate"
-	"github.com/crowdsecurity/crowdsec/pkg/types"
 )
 
 const decisionDeleteBulkSize = 256 // scientifically proven to be the best value for bulk delete
@@ -28,115 +25,16 @@ type DecisionsByScenario struct {
 	Type     string
 }
 
-func BuildDecisionRequestWithFilter(query *ent.DecisionQuery, filter map[string][]string) (*ent.DecisionQuery, error) {
-	var (
-		rng csnet.Range
-		err error
-	)
-
-	contains := true
-	/*if contains is true, return bans that *contains* the given value (value is the inner)
-	  else, return bans that are *contained* by the given value (value is the outer)*/
-
-	/*the simulated filter is a bit different : if it's not present *or* set to false, specifically exclude records with simulated to true */
-	if v, ok := filter["simulated"]; ok {
-		if v[0] == "false" {
-			query = query.Where(decision.SimulatedEQ(false))
-		}
-
-		delete(filter, "simulated")
-	} else {
-		query = query.Where(decision.SimulatedEQ(false))
-	}
-
-	for param, value := range filter {
-		switch param {
-		case "contains":
-			contains, err = strconv.ParseBool(value[0])
-			if err != nil {
-				return nil, errors.Wrapf(InvalidFilter, "invalid contains value : %s", err)
-			}
-		case "scopes", "scope": // Swagger mentions both of them, let's just support both to make sure we don't break anything
-			scopes := strings.Split(value[0], ",")
-			for i, scope := range scopes {
-				switch strings.ToLower(scope) {
-				case "ip":
-					scopes[i] = types.Ip
-				case "range":
-					scopes[i] = types.Range
-				case "country":
-					scopes[i] = types.Country
-				case "as":
-					scopes[i] = types.AS
-				}
-			}
-
-			query = query.Where(decision.ScopeIn(scopes...))
-		case "value":
-			query = query.Where(decision.ValueEQ(value[0]))
-		case "type":
-			query = query.Where(decision.TypeEQ(value[0]))
-		case "origins":
-			query = query.Where(
-				decision.OriginIn(strings.Split(value[0], ",")...),
-			)
-		case "scenarios_containing":
-			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
-			query = query.Where(decision.Or(predicates...))
-		case "scenarios_not_containing":
-			predicates := decisionPredicatesFromStr(value[0], decision.ScenarioContainsFold)
-			query = query.Where(decision.Not(
-				decision.Or(
-					predicates...,
-				),
-			))
-		case "ip", "range":
-			rng, err = csnet.NewRange(value[0])
-			if err != nil {
-				return nil, errors.Wrapf(InvalidIPOrRange, "unable to convert '%s' to int: %s", value[0], err)
-			}
-		case "limit":
-			limit, err := strconv.Atoi(value[0])
-			if err != nil {
-				return nil, errors.Wrapf(InvalidFilter, "invalid limit value : %s", err)
-			}
-
-			query = query.Limit(limit)
-		case "offset":
-			offset, err := strconv.Atoi(value[0])
-			if err != nil {
-				return nil, errors.Wrapf(InvalidFilter, "invalid offset value : %s", err)
-			}
-
-			query = query.Offset(offset)
-		case "id_gt":
-			id, err := strconv.Atoi(value[0])
-			if err != nil {
-				return nil, errors.Wrapf(InvalidFilter, "invalid id_gt value : %s", err)
-			}
-
-			query = query.Where(decision.IDGT(id))
-		}
-	}
-
-	query, err = decisionIPFilter(query, contains, rng)
-	if err != nil {
-		return nil, fmt.Errorf("fail to apply StartIpEndIpFilter: %w", err)
-	}
-
-	return query, nil
-}
-
-func (c *Client) QueryAllDecisionsWithFilters(ctx context.Context, filters map[string][]string) ([]*ent.Decision, error) {
+func (c *Client) QueryAllDecisionsWithFilters(ctx context.Context, filter map[string][]string) ([]*ent.Decision, error) {
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilGT(time.Now().UTC()),
 	)
 	// Allow a bouncer to ask for non-deduplicated results
-	if v, ok := filters["dedup"]; !ok || v[0] != "false" {
+	if v, ok := filter["dedup"]; !ok || v[0] != "false" {
 		query = query.Where(longestDecisionForScopeTypeValue)
 	}
 
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, err := applyDecisionFilter(query, filter)
 	if err != nil {
 		c.Log.Warningf("QueryAllDecisionsWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "get all decisions with filters")
@@ -153,16 +51,16 @@ func (c *Client) QueryAllDecisionsWithFilters(ctx context.Context, filters map[s
 	return data, nil
 }
 
-func (c *Client) QueryExpiredDecisionsWithFilters(ctx context.Context, filters map[string][]string) ([]*ent.Decision, error) {
+func (c *Client) QueryExpiredDecisionsWithFilters(ctx context.Context, filter map[string][]string) ([]*ent.Decision, error) {
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilLT(time.Now().UTC()),
 	)
 	// Allow a bouncer to ask for non-deduplicated results
-	if v, ok := filters["dedup"]; !ok || v[0] != "false" {
+	if v, ok := filter["dedup"]; !ok || v[0] != "false" {
 		query = query.Where(longestDecisionForScopeTypeValue)
 	}
 
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, err := applyDecisionFilter(query, filter)
 
 	query = query.Order(ent.Asc(decision.FieldID))
 
@@ -185,7 +83,7 @@ func (c *Client) QueryDecisionCountByScenario(ctx context.Context) ([]*Decisions
 		decision.UntilGT(time.Now().UTC()),
 	)
 
-	query, err := BuildDecisionRequestWithFilter(query, make(map[string][]string))
+	query, err := applyDecisionFilter(query, make(map[string][]string))
 	if err != nil {
 		c.Log.Warningf("QueryDecisionCountByScenario : %s", err)
 		return nil, errors.Wrap(QueryFail, "count all decisions with filters")
@@ -208,15 +106,15 @@ func (c *Client) QueryDecisionWithFilter(ctx context.Context, filter map[string]
 		data []*ent.Decision
 	)
 
-	decisions := c.Ent.Decision.Query().
+	query := c.Ent.Decision.Query().
 		Where(decision.UntilGTE(time.Now().UTC()))
 
-	decisions, err = BuildDecisionRequestWithFilter(decisions, filter)
+	query, err = applyDecisionFilter(query, filter)
 	if err != nil {
 		return []*ent.Decision{}, err
 	}
 
-	err = decisions.Select(
+	err = query.Select(
 		decision.FieldID,
 		decision.FieldUntil,
 		decision.FieldScenario,
@@ -263,7 +161,7 @@ func longestDecisionForScopeTypeValue(s *sql.Selector) {
 	)
 }
 
-func (c *Client) QueryExpiredDecisionsSinceWithFilters(ctx context.Context, since *time.Time, filters map[string][]string) ([]*ent.Decision, error) {
+func (c *Client) QueryExpiredDecisionsSinceWithFilters(ctx context.Context, since *time.Time, filter map[string][]string) ([]*ent.Decision, error) {
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilLT(time.Now().UTC()),
 	)
@@ -273,11 +171,11 @@ func (c *Client) QueryExpiredDecisionsSinceWithFilters(ctx context.Context, sinc
 	}
 
 	// Allow a bouncer to ask for non-deduplicated results
-	if v, ok := filters["dedup"]; !ok || v[0] != "false" {
+	if v, ok := filter["dedup"]; !ok || v[0] != "false" {
 		query = query.Where(longestDecisionForScopeTypeValue)
 	}
 
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, err := applyDecisionFilter(query, filter)
 	if err != nil {
 		c.Log.Warningf("QueryExpiredDecisionsSinceWithFilters : %s", err)
 		return []*ent.Decision{}, errors.Wrap(QueryFail, "expired decisions with filters")
@@ -294,24 +192,29 @@ func (c *Client) QueryExpiredDecisionsSinceWithFilters(ctx context.Context, sinc
 	return data, nil
 }
 
-func (c *Client) QueryNewDecisionsSinceWithFilters(ctx context.Context, since *time.Time, filters map[string][]string) ([]*ent.Decision, error) {
+func (c *Client) QueryNewDecisionsSinceWithFilters(ctx context.Context, since *time.Time, filter map[string][]string) ([]*ent.Decision, error) {
 	query := c.Ent.Decision.Query().Where(
 		decision.UntilGT(time.Now().UTC()),
 	)
 
+	errorMsg := "new decisions"
+
 	if since != nil {
 		query = query.Where(decision.CreatedAtGT(*since))
+
+		errorMsg = fmt.Sprintf("%s since %q", errorMsg, since)
 	}
 
 	// Allow a bouncer to ask for non-deduplicated results
-	if v, ok := filters["dedup"]; !ok || v[0] != "false" {
+	if v, ok := filter["dedup"]; !ok || v[0] != "false" {
 		query = query.Where(longestDecisionForScopeTypeValue)
 	}
 
-	query, err := BuildDecisionRequestWithFilter(query, filters)
+	query, err := applyDecisionFilter(query, filter)
 	if err != nil {
 		c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err)
-		return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String())
+
+		return nil, fmt.Errorf("%w: %s", QueryFail, errorMsg)
 	}
 
 	query = query.Order(ent.Asc(decision.FieldID))
@@ -319,7 +222,8 @@ func (c *Client) QueryNewDecisionsSinceWithFilters(ctx context.Context, since *t
 	data, err := query.All(ctx)
 	if err != nil {
 		c.Log.Warningf("QueryNewDecisionsSinceWithFilters : %s", err)
-		return []*ent.Decision{}, errors.Wrapf(QueryFail, "new decisions since '%s'", since.String())
+
+		return nil, fmt.Errorf("%w: %s", QueryFail, errorMsg)
 	}
 
 	return data, nil
@@ -458,7 +362,7 @@ func (c *Client) DeleteDecisions(ctx context.Context, decisions []*ent.Decision)
 	return tot, nil
 }
 
-// ExpireDecision set the expiration of a decision to now()
+// ExpireDecisionByID set the expiration of a decision to now()
 func (c *Client) ExpireDecisionByID(ctx context.Context, decisionID int) (int, []*ent.Decision, error) {
 	toUpdate, err := c.Ent.Decision.Query().Where(decision.IDEQ(decisionID)).All(ctx)
 
@@ -536,97 +440,3 @@ func (c *Client) GetActiveDecisionsTimeLeftByValue(ctx context.Context, decision
 
 	return decision.Until.Sub(time.Now().UTC()), nil
 }
-
-func decisionIPv4Filter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
-	if contains {
-		/*Decision contains {start_ip,end_ip}*/
-		return decisions.Where(decision.And(
-			decision.StartIPLTE(rng.Start.Addr),
-			decision.EndIPGTE(rng.End.Addr),
-			decision.IPSizeEQ(int64(rng.Size())))), nil
-	}
-
-	/*Decision is contained within {start_ip,end_ip}*/
-	return decisions.Where(decision.And(
-		decision.StartIPGTE(rng.Start.Addr),
-		decision.EndIPLTE(rng.End.Addr),
-		decision.IPSizeEQ(int64(rng.Size())))), nil
-}
-
-func decisionIPv6Filter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
-	/*decision contains {start_ip,end_ip}*/
-	if contains {
-		return decisions.Where(decision.And(
-			// matching addr size
-			decision.IPSizeEQ(int64(rng.Size())),
-			decision.Or(
-				// decision.start_ip < query.start_ip
-				decision.StartIPLT(rng.Start.Addr),
-				decision.And(
-					// decision.start_ip == query.start_ip
-					decision.StartIPEQ(rng.Start.Addr),
-					// decision.start_suffix <= query.start_suffix
-					decision.StartSuffixLTE(rng.Start.Sfx),
-				)),
-			decision.Or(
-				// decision.end_ip > query.end_ip
-				decision.EndIPGT(rng.End.Addr),
-				decision.And(
-					// decision.end_ip == query.end_ip
-					decision.EndIPEQ(rng.End.Addr),
-					// decision.end_suffix >= query.end_suffix
-					decision.EndSuffixGTE(rng.End.Sfx),
-				),
-			),
-		)), nil
-	}
-
-	/*decision is contained within {start_ip,end_ip}*/
-	return decisions.Where(decision.And(
-		// matching addr size
-		decision.IPSizeEQ(int64(rng.Size())),
-		decision.Or(
-			// decision.start_ip > query.start_ip
-			decision.StartIPGT(rng.Start.Addr),
-			decision.And(
-				// decision.start_ip == query.start_ip
-				decision.StartIPEQ(rng.Start.Addr),
-				// decision.start_suffix >= query.start_suffix
-				decision.StartSuffixGTE(rng.Start.Sfx),
-			)),
-		decision.Or(
-			// decision.end_ip < query.end_ip
-			decision.EndIPLT(rng.End.Addr),
-			decision.And(
-				// decision.end_ip == query.end_ip
-				decision.EndIPEQ(rng.End.Addr),
-				// decision.end_suffix <= query.end_suffix
-				decision.EndSuffixLTE(rng.End.Sfx),
-			),
-		),
-	)), nil
-}
-
-func decisionIPFilter(decisions *ent.DecisionQuery, contains bool, rng csnet.Range) (*ent.DecisionQuery, error) {
-	switch rng.Size() {
-	case 4:
-		return decisionIPv4Filter(decisions, contains, rng)
-	case 16:
-		return decisionIPv6Filter(decisions, contains, rng)
-	case 0:
-		return decisions, nil
-	default:
-		return nil, errors.Wrapf(InvalidFilter, "unknown ip size %d", rng.Size())
-	}
-}
-
-func decisionPredicatesFromStr(s string, predicateFunc func(string) predicate.Decision) []predicate.Decision {
-	words := strings.Split(s, ",")
-	predicates := make([]predicate.Decision, len(words))
-
-	for i, word := range words {
-		predicates[i] = predicateFunc(word)
-	}
-
-	return predicates
-}