diff --git a/cmd/crowdsec-cli/cliitem/metrics.go b/cmd/crowdsec-cli/cliitem/metrics.go index 4999ea380..1b63f3bfe 100644 --- a/cmd/crowdsec-cli/cliitem/metrics.go +++ b/cmd/crowdsec-cli/cliitem/metrics.go @@ -15,6 +15,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/cwhub" + "github.com/crowdsecurity/crowdsec/pkg/metrics" ) func showMetrics(prometheusURL string, hub *cwhub.Hub, hubItem *cwhub.Item, wantColor string) error { @@ -102,36 +103,27 @@ func getParserMetric(url string, itemName string) (map[string]map[string]int, er ival := int(fval) switch fam.Name { - case "cs_reader_hits_total": - if _, ok := stats[source]; !ok { - stats[source] = make(map[string]int) - stats[source]["parsed"] = 0 - stats[source]["reads"] = 0 - stats[source]["unparsed"] = 0 - stats[source]["hits"] = 0 - } - stats[source]["reads"] += ival - case "cs_parser_hits_ok_total": + case metrics.GlobalParserHitsOkMetricName: if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["parsed"] += ival - case "cs_parser_hits_ko_total": + case metrics.GlobalParserHitsKoMetricName: if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["unparsed"] += ival - case "cs_node_hits_total": + case metrics.NodesHitsMetricName: if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["hits"] += ival - case "cs_node_hits_ok_total": + case metrics.NodesHitsOkMetricName: if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } stats[source]["parsed"] += ival - case "cs_node_hits_ko_total": + case metrics.NodesHitsKoMetricName: if _, ok := stats[source]; !ok { stats[source] = make(map[string]int) } @@ -194,15 +186,15 @@ func getScenarioMetric(url string, itemName string) (map[string]int, error) { ival := int(fval) switch fam.Name { - case "cs_bucket_created_total": + case metrics.BucketsInstantiationMetricName: stats["instantiation"] += ival - case "cs_buckets": + case metrics.BucketsCurrentCountMetricName: stats["curr_count"] += ival - case "cs_bucket_overflowed_total": + case metrics.BucketsOverflowMetricName: stats["overflow"] += ival - case "cs_bucket_poured_total": + case metrics.BucketPouredMetricName: stats["pour"] += ival - case "cs_bucket_underflowed_total": + case metrics.BucketsUnderflowMetricName: stats["underflow"] += ival default: continue diff --git a/cmd/crowdsec-cli/climachine/inspect.go b/cmd/crowdsec-cli/climachine/inspect.go index 1bd85903d..baec6d5a4 100644 --- a/cmd/crowdsec-cli/climachine/inspect.go +++ b/cmd/crowdsec-cli/climachine/inspect.go @@ -6,6 +6,7 @@ import ( "errors" "fmt" "io" + "sort" "github.com/fatih/color" "github.com/jedib0t/go-pretty/v6/table" @@ -16,8 +17,30 @@ import ( "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cstable" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/database/ent" + "github.com/crowdsecurity/crowdsec/pkg/models" ) +// metricsInfo contains processed metrics data for JSON output +type metricsInfo struct { + Acquisition []acquisitionMetric `json:"acquisition,omitempty"` + Parsers []parserMetric `json:"parsers,omitempty"` +} + +type acquisitionMetric struct { + Source string `json:"source"` + Read int `json:"read"` + Parsed int `json:"parsed"` + Unparsed int `json:"unparsed"` +} + +type parserMetric struct { + Source string `json:"source"` + Parser string `json:"parser"` + Stage string `json:"stage"` + Parsed int `json:"parsed"` + Unparsed int `json:"unparsed"` +} + func (cli *cliMachines) inspectHubHuman(out io.Writer, machine *ent.Machine) { state := machine.Hubstate @@ -84,18 +107,278 @@ func (cli *cliMachines) inspectHuman(out io.Writer, machine *ent.Machine) { fmt.Fprintln(out, t.Render()) } -func (cli *cliMachines) inspect(machine *ent.Machine) error { +func (cli *cliMachines) inspectMetrics(out io.Writer, metrics []*ent.Metric) { + // Create first table for global acquisition metrics (read, global_parsed, global_unparsed) + cli.inspectAcquisitionMetrics(out, metrics) + + // Create second table for parser-specific metrics (parsed, unparsed) + cli.inspectParserMetrics(out, metrics) +} + +// processAcquisitionMetrics aggregates acquisition metrics data from raw metrics +func (cli *cliMachines) processAcquisitionMetrics(metrics []*ent.Metric) []acquisitionMetric { + aggregatedMetrics := make(map[string]map[string]int) + + for _, metric := range metrics { + var payload models.LogProcessorsMetrics + + if err := json.Unmarshal([]byte(metric.Payload), &payload); err != nil { + continue + } + + for _, detailedMetric := range payload.Metrics { + if detailedMetric.Items == nil { + continue + } + + for _, item := range detailedMetric.Items { + if item.Name == nil || item.Labels == nil || item.Value == nil { + continue + } + + metricName := *item.Name + + if metricName != "read" && metricName != "global_parsed" && metricName != "global_unparsed" { + continue + } + + datasourceType, hasDataSourceType := item.Labels["datasource_type"] + source, hasSource := item.Labels["source"] + acquisType, hasAcquisType := item.Labels["acquis_type"] + + if !hasDataSourceType || !hasSource || !hasAcquisType { + continue + } + + sourceID := fmt.Sprintf("%s:%s (%s)", datasourceType, source, acquisType) + + if aggregatedMetrics[sourceID] == nil { + aggregatedMetrics[sourceID] = make(map[string]int) + } + + var columnName string + switch metricName { + case "read": + columnName = "read" + case "global_parsed": + columnName = "parsed" + case "global_unparsed": + columnName = "unparsed" + } + + aggregatedMetrics[sourceID][columnName] += int(*item.Value) + } + } + } + + // Convert to slice and sort + var result []acquisitionMetric + var sources []string + for source := range aggregatedMetrics { + sources = append(sources, source) + } + sort.Strings(sources) + + for _, source := range sources { + metrics := aggregatedMetrics[source] + result = append(result, acquisitionMetric{ + Source: source, + Read: metrics["read"], + Parsed: metrics["parsed"], + Unparsed: metrics["unparsed"], + }) + } + + return result +} + +func (cli *cliMachines) inspectAcquisitionMetrics(out io.Writer, metrics []*ent.Metric) { + acquisitionMetrics := cli.processAcquisitionMetrics(metrics) + + if len(acquisitionMetrics) == 0 { + return + } + + t := cstable.New(out, cli.cfg().Cscli.Color).Writer + + t.SetTitle("Acquisition Metrics") + t.AppendHeader(table.Row{"Source", "Read", "Parsed", "Unparsed"}) + + t.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: true}, + }) + + for _, metric := range acquisitionMetrics { + t.AppendRow(table.Row{ + metric.Source, + metric.Read, + metric.Parsed, + metric.Unparsed, + }) + } + + fmt.Fprintln(out, t.Render()) +} + +// processParserMetrics aggregates parser metrics data from raw metrics +func (cli *cliMachines) processParserMetrics(metrics []*ent.Metric) []parserMetric { + type parserInfo struct { + stage string + name string + metrics map[string]int + } + aggregatedMetrics := make(map[string][]parserInfo) + + for _, metric := range metrics { + var payload models.LogProcessorsMetrics + + if err := json.Unmarshal([]byte(metric.Payload), &payload); err != nil { + continue + } + + for _, detailedMetric := range payload.Metrics { + for _, item := range detailedMetric.Items { + if item.Name == nil || item.Labels == nil || item.Value == nil { + continue + } + + metricName := *item.Name + + if metricName != "parsed" && metricName != "unparsed" { + continue + } + + datasourceType := item.Labels["datasource_type"] + source := item.Labels["source"] + acquisType := item.Labels["acquis_type"] + parserName := item.Labels["parser_name"] + parserStage := item.Labels["parser_stage"] + + var sourceID string + if datasourceType == "" && source == "" && acquisType == "" { + // Postoverflows have lost the acquisition information, treat as a special case + sourceID = "Postoverflows" + } else { + sourceID = fmt.Sprintf("%s:%s (%s)", datasourceType, source, acquisType) + } + + var foundParser *parserInfo + for i := range aggregatedMetrics[sourceID] { + if aggregatedMetrics[sourceID][i].name == parserName { + foundParser = &aggregatedMetrics[sourceID][i] + break + } + } + + if foundParser == nil { + newParser := parserInfo{ + stage: parserStage, + name: parserName, + metrics: make(map[string]int), + } + aggregatedMetrics[sourceID] = append(aggregatedMetrics[sourceID], newParser) + foundParser = &aggregatedMetrics[sourceID][len(aggregatedMetrics[sourceID])-1] + } + + foundParser.metrics[metricName] += int(*item.Value) + } + } + } + + var result []parserMetric + var sources []string + for source := range aggregatedMetrics { + sources = append(sources, source) + } + + sort.Slice(sources, func(i, j int) bool { + // Always put "Postoverflows" last + if sources[i] == "Postoverflows" { + return false + } + if sources[j] == "Postoverflows" { + return true + } + return sources[i] < sources[j] + }) + + for _, source := range sources { + parsers := aggregatedMetrics[source] + + // Sort parsers first by stage, then alphabetically by name + sort.Slice(parsers, func(i, j int) bool { + if parsers[i].stage != parsers[j].stage { + return parsers[i].stage < parsers[j].stage + } + return parsers[i].name < parsers[j].name + }) + + for _, parser := range parsers { + result = append(result, parserMetric{ + Source: source, + Parser: parser.name, + Stage: parser.stage, + Parsed: parser.metrics["parsed"], + Unparsed: parser.metrics["unparsed"], + }) + } + } + + return result +} + +func (cli *cliMachines) inspectParserMetrics(out io.Writer, metrics []*ent.Metric) { + parserMetrics := cli.processParserMetrics(metrics) + + if len(parserMetrics) == 0 { + return + } + + t := cstable.New(out, cli.cfg().Cscli.Color).Writer + + t.SetTitle("Parser Metrics") + t.AppendHeader(table.Row{"Source", "Parser", "Parsed", "Unparsed"}) + + t.SetColumnConfigs([]table.ColumnConfig{ + {Number: 1, AutoMerge: true}, + }) + + for _, metric := range parserMetrics { + t.AppendRow(table.Row{ + metric.Source, + fmt.Sprintf("%s/%s", metric.Stage, metric.Parser), + metric.Parsed, + metric.Unparsed, + }) + } + + fmt.Fprintln(out, t.Render()) +} + +func (cli *cliMachines) inspect(machine *ent.Machine, metrics []*ent.Metric) error { out := color.Output outputFormat := cli.cfg().Cscli.Output switch outputFormat { case "human": cli.inspectHuman(out, machine) + cli.inspectMetrics(out, metrics) case "json": enc := json.NewEncoder(out) enc.SetIndent("", " ") - if err := enc.Encode(newMachineInfo(machine)); err != nil { + machineData := newMachineInfo(machine) + metricsData := cli.processMetricsForJSON(metrics) + + result := struct { + machineInfo + Metrics metricsInfo `json:"metrics,omitempty"` + }{ + machineInfo: machineData, + Metrics: metricsData, + } + + if err := enc.Encode(result); err != nil { return errors.New("failed to serialize") } @@ -107,6 +390,14 @@ func (cli *cliMachines) inspect(machine *ent.Machine) error { return nil } +func (cli *cliMachines) processMetricsForJSON(metrics []*ent.Metric) metricsInfo { + result := metricsInfo{ + Acquisition: cli.processAcquisitionMetrics(metrics), + Parsers: cli.processParserMetrics(metrics), + } + return result +} + func (cli *cliMachines) inspectHub(machine *ent.Machine) error { out := color.Output @@ -173,7 +464,11 @@ func (cli *cliMachines) newInspectCmd() *cobra.Command { return cli.inspectHub(machine) } - return cli.inspect(machine) + machineMetrics, err := cli.db.GetLPUsageMetricsByMachineID(ctx, machineID, false) + if err != nil { + return fmt.Errorf("unable to read machine metrics '%s': %w", machineID, err) + } + return cli.inspect(machine, machineMetrics) }, } diff --git a/cmd/crowdsec-cli/climetrics/statlapidecision.go b/cmd/crowdsec-cli/climetrics/statlapidecision.go index 3371cb0e8..1ef9f4102 100644 --- a/cmd/crowdsec-cli/climetrics/statlapidecision.go +++ b/cmd/crowdsec-cli/climetrics/statlapidecision.go @@ -8,6 +8,7 @@ import ( "github.com/jedib0t/go-pretty/v6/table" "github.com/crowdsecurity/crowdsec/cmd/crowdsec-cli/cstable" + "github.com/crowdsecurity/crowdsec/pkg/metrics" ) type statLapiDecision map[string]struct { @@ -31,9 +32,9 @@ func (s statLapiDecision) Process(bouncer, fam string, val int) { x := s[bouncer] switch fam { - case "cs_lapi_decisions_ko_total": + case metrics.LapiNilDecisionsMetricName: x.Empty += val - case "cs_lapi_decisions_ok_total": + case metrics.LapiNonNilDecisionsMetricName: x.NonEmpty += val } diff --git a/cmd/crowdsec-cli/climetrics/store.go b/cmd/crowdsec-cli/climetrics/store.go index 6c4024479..58b41669b 100644 --- a/cmd/crowdsec-cli/climetrics/store.go +++ b/cmd/crowdsec-cli/climetrics/store.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/database" + "github.com/crowdsecurity/crowdsec/pkg/metrics" ) type metricSection interface { @@ -161,63 +162,63 @@ func (ms metricStore) processPrometheusMetrics(result []*prom2json.Family) { // // buckets // - case "cs_bucket_created_total": + case metrics.BucketsInstantiationMetricName: mBucket.Process(name, "instantiation", ival) - case "cs_buckets": + case metrics.BucketsCurrentCountMetricName: mBucket.Process(name, "curr_count", ival) - case "cs_bucket_overflowed_total": + case metrics.BucketsOverflowMetricName: mBucket.Process(name, "overflow", ival) - case "cs_bucket_poured_total": + case metrics.BucketPouredMetricName: mBucket.Process(name, "pour", ival) mAcquis.Process(source, "pour", ival) - case "cs_bucket_underflowed_total": + case metrics.BucketsUnderflowMetricName: mBucket.Process(name, "underflow", ival) // // parsers // - case "cs_parser_hits_total": + case metrics.GlobalParserHitsMetricName: mAcquis.Process(source, "reads", ival) - case "cs_parser_hits_ok_total": + case metrics.GlobalParserHitsOkMetricName: mAcquis.Process(source, "parsed", ival) - case "cs_parser_hits_ko_total": + case metrics.GlobalParserHitsKoMetricName: mAcquis.Process(source, "unparsed", ival) - case "cs_node_hits_total": + case metrics.NodesHitsMetricName: mParser.Process(name, "hits", ival) - case "cs_node_hits_ok_total": + case metrics.NodesHitsOkMetricName: mParser.Process(name, "parsed", ival) - case "cs_node_hits_ko_total": + case metrics.NodesHitsKoMetricName: mParser.Process(name, "unparsed", ival) // // whitelists // - case "cs_node_wl_hits_total": + case metrics.NodesWlHitsMetricName: mWhitelist.Process(name, reason, "hits", ival) - case "cs_node_wl_hits_ok_total": + case metrics.NodesWlHitsOkMetricName: mWhitelist.Process(name, reason, "whitelisted", ival) // track as well whitelisted lines at acquis level mAcquis.Process(source, "whitelisted", ival) // // lapi // - case "cs_lapi_route_requests_total": + case metrics.LapiRouteHitsMetricName: mLapi.Process(route, method, ival) - case "cs_lapi_machine_requests_total": + case metrics.LapiMachineHitsMetricName: mLapiMachine.Process(machine, route, method, ival) - case "cs_lapi_bouncer_requests_total": + case metrics.LapiBouncerHitsMetricName: mLapiBouncer.Process(bouncer, route, method, ival) - case "cs_lapi_decisions_ko_total", "cs_lapi_decisions_ok_total": + case metrics.LapiNilDecisionsMetricName, metrics.LapiNonNilDecisionsMetricName: mLapiDecision.Process(bouncer, fam.Name, ival) // // decisions // - case "cs_active_decisions": + case metrics.GlobalActiveDecisionsMetricName: mDecision.Process(reason, origin, action, ival) - case "cs_alerts": + case metrics.GlobalAlertsMetricName: mAlert.Process(reason, ival) // // stash // - case "cs_cache_size": + case metrics.CacheMetricName: mStash.Process(name, mtype, ival) // // appsec diff --git a/cmd/crowdsec/crowdsec.go b/cmd/crowdsec/crowdsec.go index 1f74a5a1d..780e11cad 100644 --- a/cmd/crowdsec/crowdsec.go +++ b/cmd/crowdsec/crowdsec.go @@ -12,13 +12,13 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -168,7 +168,7 @@ func startLPMetrics(cConfig *csconfig.Config, apiClient *apiclient.ApiClient, hu if cConfig.Prometheus != nil && cConfig.Prometheus.Enabled { aggregated := false - if cConfig.Prometheus.Level == configuration.CFG_METRICS_AGGREGATE { + if cConfig.Prometheus.Level == metrics.MetricsLevelAggregated { aggregated = true } diff --git a/cmd/crowdsec/lpmetrics.go b/cmd/crowdsec/lpmetrics.go index 48eeaae91..fe5ec00c9 100644 --- a/cmd/crowdsec/lpmetrics.go +++ b/cmd/crowdsec/lpmetrics.go @@ -4,8 +4,14 @@ import ( "context" "errors" "net/http" + "regexp" + "slices" + "strings" "time" + "github.com/prometheus/client_golang/prometheus" + io_prometheus_client "github.com/prometheus/client_model/go" + "github.com/sirupsen/logrus" "gopkg.in/tomb.v2" @@ -17,11 +23,21 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/apiclient" "github.com/crowdsecurity/crowdsec/pkg/cwhub" "github.com/crowdsecurity/crowdsec/pkg/fflag" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/models" ) const lpMetricsDefaultInterval = 30 * time.Minute +var childNodeExcludeRegexp = regexp.MustCompile("^child-") + +// This is not stored in the metrics provider because it gets recreated during a reload +// Which would make us lose the last values of the metrics, and mess up the delta for the next run because the prometheus metrics themselves are not reset +// There's only a single instance of MetricsProvider, so no need to mutex or anything +// This used to store the last collected value of a metric to compute the delta before sending it +// Key is a concatenation of all labels +var metricsLastValues = make(map[string]float64) + // MetricsProvider collects metrics from the LP and sends them to the LAPI type MetricsProvider struct { apic *apiclient.ApiClient @@ -40,6 +56,10 @@ type staticMetrics struct { hubState models.HubItems } +// Key is the prom label +// Value is the name that will be used in the metrics payload +type labelsMapping map[string]string + func getHubState(hub *cwhub.Hub) models.HubItems { ret := models.HubItems{} @@ -97,6 +117,173 @@ func NewMetricsProvider(apic *apiclient.ApiClient, interval time.Duration, logge } } +func getLabelValue(labels []*io_prometheus_client.LabelPair, key string) string { + for _, label := range labels { + if label.GetName() == key { + return label.GetValue() + } + } + return "" +} + +func getDeltaKey(metricName string, labels []*io_prometheus_client.LabelPair) string { + // Create a key from the labels to use as a map key + // This is used to store the last value of the metric to compute the delta + parts := make([]string, 0, len(labels)+1) + parts = append(parts, metricName) + sortedLabels := slices.Clone(labels) + slices.SortFunc(sortedLabels, func(a, b *io_prometheus_client.LabelPair) int { + return strings.Compare(a.GetName(), b.GetName()) + }) + for _, label := range sortedLabels { + parts = append(parts, label.GetName()+label.GetValue()) + } + return strings.Join(parts, "") +} + +func shouldIgnoreMetric(exclude map[string]*regexp.Regexp, promLabels []*io_prometheus_client.LabelPair) bool { + for labelKey, regex := range exclude { + labelValue := getLabelValue(promLabels, labelKey) + if labelValue == "" { + continue + } + if regex.MatchString(labelValue) { + return true + } + } + return false +} + +func (m *MetricsProvider) gatherPromMetrics(metricsName []string, labelsMap labelsMapping, exclude map[string]*regexp.Regexp, metricName string, unitType string) []*models.MetricsDetailItem { + items := make([]*models.MetricsDetailItem, 0) + + promMetrics, err := prometheus.DefaultGatherer.Gather() + if err != nil { + m.logger.Errorf("failed to gather prometheus metrics: %s", err) + return nil + } + + for _, metricFamily := range promMetrics { + if !slices.Contains(metricsName, metricFamily.GetName()) { + continue + } + for _, metric := range metricFamily.GetMetric() { + promLabels := metric.GetLabel() + + if shouldIgnoreMetric(exclude, promLabels) { + continue + } + + deltaKey := getDeltaKey(metricFamily.GetName(), promLabels) + metricsLabels := make(map[string]string) + + for labelKey, labelValue := range labelsMap { + metricsLabels[labelValue] = getLabelValue(promLabels, labelKey) + } + + currentValue := metric.GetCounter().GetValue() + value := currentValue + + if lastValue, ok := metricsLastValues[deltaKey]; ok { + value -= lastValue + if value < 0 { + m.logger.Warnf("negative delta for metric %s (labels: %+v), resetting to 0. This is probably a bug.", metricName, metricsLabels) + value = 0 + } + } + metricsLastValues[deltaKey] = currentValue + + if value == 0 { + continue + } + + item := &models.MetricsDetailItem{ + Name: ptr.Of(metricName), + Unit: ptr.Of(unitType), + Labels: metricsLabels, + Value: ptr.Of(value), + } + m.logger.Debugf("Gathered metric: %s, item: %+v", metricFamily.GetName(), item) + items = append(items, item) + } + } + + return items +} + +func (m *MetricsProvider) getAcquisitionMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics(metrics.AcquisitionMetricsNames, labelsMapping{ + "datasource_type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + }, nil, "read", "line") +} + +func (m *MetricsProvider) getParserGlobalOkMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.GlobalParserHitsOkMetricName}, labelsMapping{ + "type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + }, nil, "global_parsed", "line") +} + +func (m *MetricsProvider) getParserGlobalKoMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.GlobalParserHitsKoMetricName}, labelsMapping{ + "type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + }, nil, "global_unparsed", "line") +} + +func (m *MetricsProvider) getParserSuccessMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.NodesHitsOkMetricName}, labelsMapping{ + "type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + "name": "parser_name", + "stage": "parser_stage", + }, map[string]*regexp.Regexp{ + "name": childNodeExcludeRegexp, + }, "parsed", "line", + ) +} + +func (m *MetricsProvider) getParserFailureMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.NodesHitsKoMetricName}, labelsMapping{ + "type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + "name": "parser_name", + "stage": "parser_stage", + }, map[string]*regexp.Regexp{ + "name": childNodeExcludeRegexp, + }, "unparsed", "line", + ) +} + +func (m *MetricsProvider) getParserWhitelistMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.NodesWlHitsOkMetricName}, labelsMapping{ + "type": "datasource_type", + "source": "source", + "acquis_type": "acquis_type", + "name": "whitelist_name", + "stage": "whitelist_stage", + }, nil, "whitelisted", "event", + ) +} + +func (m *MetricsProvider) getAppsecProcessedMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.AppsecReqCounterMetricName}, labelsMapping{ + "appsec_engine": "appsec_engine", + }, nil, "appsec_processed", "request") +} + +func (m *MetricsProvider) getAppsecBlockedMetrics() []*models.MetricsDetailItem { + return m.gatherPromMetrics([]string{metrics.AppsecBlockCounterMetricName}, labelsMapping{ + "appsec_engine": "appsec_engine", + }, nil, "appsec_blocked", "request") +} + func (m *MetricsProvider) metricsPayload() *models.AllMetrics { os := &models.OSversion{ Name: ptr.Of(m.static.osName), @@ -125,6 +312,53 @@ func (m *MetricsProvider) metricsPayload() *models.AllMetrics { Items: make([]*models.MetricsDetailItem, 0), }) + /* Acquisition metrics */ + /*{"name": "read", "value": 10, "unit": "line", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ + /* Parser metrics */ + /*{"name": "parsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ + /*{"name": "unparsed", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ + /*{"name": "whitelisted", labels: {"datasource_type": "file", "source":"/var/log/auth.log"}}*/ + + acquisitionMetrics := m.getAcquisitionMetrics() + if len(acquisitionMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, acquisitionMetrics...) + } + + parserSuccessMetrics := m.getParserSuccessMetrics() + if len(parserSuccessMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, parserSuccessMetrics...) + } + + parserFailureMetrics := m.getParserFailureMetrics() + if len(parserFailureMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, parserFailureMetrics...) + } + + parserWhitelistMetrics := m.getParserWhitelistMetrics() + if len(parserWhitelistMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, parserWhitelistMetrics...) + } + + globalParsedMetrics := m.getParserGlobalOkMetrics() + if len(globalParsedMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, globalParsedMetrics...) + } + + globalUnparsedMetrics := m.getParserGlobalKoMetrics() + if len(globalUnparsedMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, globalUnparsedMetrics...) + } + + appsecProcessedMetrics := m.getAppsecProcessedMetrics() + if len(appsecProcessedMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, appsecProcessedMetrics...) + } + + appsecBlockedMetrics := m.getAppsecBlockedMetrics() + if len(appsecBlockedMetrics) > 0 { + met.Metrics[0].Items = append(met.Metrics[0].Items, appsecBlockedMetrics...) + } + return &models.AllMetrics{ LogProcessors: []*models.LogProcessorsMetrics{met}, } @@ -158,13 +392,12 @@ func (m *MetricsProvider) Run(ctx context.Context, myTomb *tomb.Tomb) error { return nil } - met := m.metricsPayload() - ticker := time.NewTicker(1) // Send on start for { select { case <-ticker.C: + met := m.metricsPayload() m.sendMetrics(ctx, met) ticker.Reset(m.interval) case <-myTomb.Dying(): diff --git a/cmd/crowdsec/metrics.go b/cmd/crowdsec/metrics.go index ff280fc35..872192697 100644 --- a/cmd/crowdsec/metrics.go +++ b/cmd/crowdsec/metrics.go @@ -9,98 +9,12 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/go-cs-lib/trace" - "github.com/crowdsecurity/go-cs-lib/version" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" - "github.com/crowdsecurity/crowdsec/pkg/apiserver/controllers/v1" "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" - leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" - "github.com/crowdsecurity/crowdsec/pkg/parser" -) - -// Prometheus - -var globalParserHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_parser_hits_total", - Help: "Total events entered the parser.", - }, - []string{"source", "type"}, -) - -var globalParserHitsOk = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_parser_hits_ok_total", - Help: "Total events were successfully parsed.", - }, - []string{"source", "type"}, -) - -var globalParserHitsKo = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_parser_hits_ko_total", - Help: "Total events were unsuccessfully parsed.", - }, - []string{"source", "type"}, -) - -var globalBucketPourKo = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "cs_bucket_pour_ko_total", - Help: "Total events were not poured in a bucket.", - }, -) - -var globalBucketPourOk = prometheus.NewCounter( - prometheus.CounterOpts{ - Name: "cs_bucket_pour_ok_total", - Help: "Total events were poured in at least one bucket.", - }, -) - -var globalCsInfo = prometheus.NewGauge( - prometheus.GaugeOpts{ - Name: "cs_info", - Help: "Information about Crowdsec.", - ConstLabels: prometheus.Labels{"version": version.String()}, - }, -) - -var globalActiveDecisions = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_active_decisions", - Help: "Number of active decisions.", - }, - []string{"reason", "origin", "action"}, -) - -var globalAlerts = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_alerts", - Help: "Number of alerts (excluding CAPI).", - }, - []string{"reason"}, -) - -var globalParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent parsing a line", - Name: "cs_parsing_time_seconds", - Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, - }, - []string{"type", "source"}, -) - -var globalPourHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "cs_bucket_pour_seconds", - Help: "Time spent pouring an event to buckets.", - Buckets: []float64{0.001, 0.002, 0.005, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05}, - }, - []string{"type", "source"}, + "github.com/crowdsecurity/crowdsec/pkg/metrics" ) func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.HandlerFunc { @@ -128,13 +42,13 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha return } - globalActiveDecisions.Reset() + metrics.GlobalActiveDecisions.Reset() for _, d := range decisions { - globalActiveDecisions.With(prometheus.Labels{"reason": d.Scenario, "origin": d.Origin, "action": d.Type}).Set(float64(d.Count)) + metrics.GlobalActiveDecisions.With(prometheus.Labels{"reason": d.Scenario, "origin": d.Origin, "action": d.Type}).Set(float64(d.Count)) } - globalAlerts.Reset() + metrics.GlobalAlerts.Reset() alertsFilter := map[string][]string{ "include_capi": {"false"}, @@ -149,7 +63,7 @@ func computeDynamicMetrics(next http.Handler, dbClient *database.Client) http.Ha } for k, v := range alerts { - globalAlerts.With(prometheus.Labels{"reason": k}).Set(float64(v)) + metrics.GlobalAlerts.With(prometheus.Labels{"reason": k}).Set(float64(v)) } next.ServeHTTP(w, r) @@ -161,27 +75,9 @@ func registerPrometheus(config *csconfig.PrometheusCfg) { return } - // Registering prometheus - // If in aggregated mode, do not register events associated with a source, to keep the cardinality low - if config.Level == configuration.CFG_METRICS_AGGREGATE { - log.Infof("Loading aggregated prometheus collectors") - prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, - globalCsInfo, globalParsingHistogram, globalPourHistogram, - leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, - v1.LapiRouteHits, - leaky.BucketsCurrentCount, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, parser.NodesWlHitsOk, parser.NodesWlHits, - ) - } else { - log.Infof("Loading prometheus collectors") - prometheus.MustRegister(globalParserHits, globalParserHitsOk, globalParserHitsKo, - parser.NodesHits, parser.NodesHitsOk, parser.NodesHitsKo, - globalCsInfo, globalParsingHistogram, globalPourHistogram, - v1.LapiRouteHits, v1.LapiMachineHits, v1.LapiBouncerHits, v1.LapiNilDecisions, v1.LapiNonNilDecisions, v1.LapiResponseTime, - leaky.BucketsPour, leaky.BucketsUnderflow, leaky.BucketsCanceled, leaky.BucketsInstantiation, leaky.BucketsOverflow, leaky.BucketsCurrentCount, - globalActiveDecisions, globalAlerts, parser.NodesWlHitsOk, parser.NodesWlHits, - cache.CacheMetrics, exprhelpers.RegexpCacheMetrics, - ) + if err := metrics.RegisterMetrics(config.Level); err != nil { + log.Errorf("Error registering prometheus metrics: %v", err) + return } } diff --git a/cmd/crowdsec/parse.go b/cmd/crowdsec/parse.go index 26eae66be..04a6afd9e 100644 --- a/cmd/crowdsec/parse.go +++ b/cmd/crowdsec/parse.go @@ -6,6 +6,7 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -31,7 +32,7 @@ func runParse(input chan types.Event, output chan types.Event, parserCTX parser. log.Errorf("empty event.Line.Module field, the acquisition module must set it ! : %+v", event.Line) continue } - globalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() + metrics.GlobalParserHits.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() startParsing := time.Now() /* parse the log using magic */ @@ -40,13 +41,13 @@ func runParse(input chan types.Event, output chan types.Event, parserCTX parser. log.Errorf("failed parsing: %v", err) } elapsed := time.Since(startParsing) - globalParsingHistogram.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Observe(elapsed.Seconds()) + metrics.GlobalParsingHistogram.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Observe(elapsed.Seconds()) if !parsed.Process { - globalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() + metrics.GlobalParserHitsKo.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module, "acquis_type": event.Line.Labels["type"]}).Inc() log.Debugf("Discarding line %+v", parsed) continue } - globalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module}).Inc() + metrics.GlobalParserHitsOk.With(prometheus.Labels{"source": event.Line.Src, "type": event.Line.Module, "acquis_type": event.Line.Labels["type"]}).Inc() if parsed.Whitelisted { log.Debugf("event whitelisted, discard") continue diff --git a/cmd/crowdsec/pour.go b/cmd/crowdsec/pour.go index 4c83b65bd..06eb246b5 100644 --- a/cmd/crowdsec/pour.go +++ b/cmd/crowdsec/pour.go @@ -9,6 +9,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" leaky "github.com/crowdsecurity/crowdsec/pkg/leakybucket" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -51,12 +52,12 @@ func runPour(input chan types.Event, holders []leaky.BucketFactory, buckets *lea } elapsed := time.Since(startTime) - globalPourHistogram.With(prometheus.Labels{"type": parsed.Line.Module, "source": parsed.Line.Src}).Observe(elapsed.Seconds()) + metrics.GlobalPourHistogram.With(prometheus.Labels{"type": parsed.Line.Module, "source": parsed.Line.Src}).Observe(elapsed.Seconds()) if poured { - globalBucketPourOk.Inc() + metrics.GlobalBucketPourOk.Inc() } else { - globalBucketPourKo.Inc() + metrics.GlobalBucketPourKo.Inc() } if parsed.MarshaledTime != "" { diff --git a/go.mod b/go.mod index 1e6e236e0..60093f2c0 100644 --- a/go.mod +++ b/go.mod @@ -93,6 +93,8 @@ require ( k8s.io/apiserver v0.28.4 ) +require github.com/gin-contrib/gzip v1.2.3 + require ( ariga.io/atlas v0.31.1-0.20250212144724-069be8033e83 // indirect github.com/Masterminds/goutils v1.1.1 // indirect @@ -102,9 +104,9 @@ require ( github.com/asaskevich/govalidator v0.0.0-20200907205600-7a23bdc65eef // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bmatcuk/doublestar v1.3.4 // indirect - github.com/bytedance/sonic v1.12.6 // indirect - github.com/bytedance/sonic/loader v0.2.1 // indirect - github.com/cloudwego/base64x v0.1.4 // indirect + github.com/bytedance/sonic v1.13.2 // indirect + github.com/bytedance/sonic/loader v0.2.4 // indirect + github.com/cloudwego/base64x v0.1.5 // indirect github.com/cloudwego/iasm v0.2.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -113,8 +115,8 @@ require ( github.com/distribution/reference v0.6.0 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect - github.com/gabriel-vasile/mimetype v1.4.7 // indirect - github.com/gin-contrib/sse v0.1.0 // indirect + github.com/gabriel-vasile/mimetype v1.4.8 // indirect + github.com/gin-contrib/sse v1.0.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -127,9 +129,9 @@ require ( github.com/go-openapi/spec v0.20.0 // indirect github.com/go-playground/locales v0.14.1 // indirect github.com/go-playground/universal-translator v0.18.1 // indirect - github.com/go-playground/validator/v10 v10.23.0 // indirect + github.com/go-playground/validator/v10 v10.26.0 // indirect github.com/go-stack/stack v1.8.0 // indirect - github.com/goccy/go-json v0.10.4 // indirect + github.com/goccy/go-json v0.10.5 // indirect github.com/golang/glog v1.2.4 // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/go-cmp v0.6.0 // indirect @@ -151,7 +153,7 @@ require ( github.com/json-iterator/go v1.1.12 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/compress v1.17.9 // indirect - github.com/klauspost/cpuid/v2 v2.2.9 // indirect + github.com/klauspost/cpuid/v2 v2.2.10 // indirect github.com/leodido/go-urn v1.4.0 // indirect github.com/lufia/plan9stats v0.0.0-20211012122336-39d0f177ccd0 // indirect github.com/magefile/mage v1.15.1-0.20241126214340-bdc92f694516 // indirect @@ -211,7 +213,7 @@ require ( go.opentelemetry.io/otel/sdk v1.28.0 // indirect go.opentelemetry.io/otel/trace v1.28.0 // indirect go.uber.org/atomic v1.10.0 // indirect - golang.org/x/arch v0.12.0 // indirect + golang.org/x/arch v0.15.0 // indirect golang.org/x/term v0.32.0 // indirect golang.org/x/time v0.6.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index b2a61ec03..cfdeee5e6 100644 --- a/go.sum +++ b/go.sum @@ -74,9 +74,13 @@ github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMU github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bytedance/sonic v1.12.6 h1:/isNmCUF2x3Sh8RAp/4mh4ZGkcFAX/hLrzrK3AvpRzk= github.com/bytedance/sonic v1.12.6/go.mod h1:B8Gt/XvtZ3Fqj+iSKMypzymZxw/FVwgIGKzMzT9r/rk= +github.com/bytedance/sonic v1.13.2 h1:8/H1FempDZqC4VqjptGo14QQlJx8VdZJegxs6wwfqpQ= +github.com/bytedance/sonic v1.13.2/go.mod h1:o68xyaF9u2gvVBuGHPlUVCy+ZfmNNO5ETf1+KgkJhz4= github.com/bytedance/sonic/loader v0.1.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= github.com/bytedance/sonic/loader v0.2.1 h1:1GgorWTqf12TA8mma4DDSbaQigE2wOgQo7iCjjJv3+E= github.com/bytedance/sonic/loader v0.2.1/go.mod h1:ncP89zfokxS5LZrJxl5z0UJcsk4M4yY2JpfqGeCtNLU= +github.com/bytedance/sonic/loader v0.2.4 h1:ZWCw4stuXUsn1/+zQDqeE7JKP+QO47tz7QCNan80NzY= +github.com/bytedance/sonic/loader v0.2.4/go.mod h1:N8A3vUdtUebEY2/VQC0MyhYeKUFosQU6FxH2JmUe6VI= github.com/cenkalti/backoff/v4 v4.3.0 h1:MyRJ/UdXutAwSAT+s3wNd7MfTIcy71VQueUuFK343L8= github.com/cenkalti/backoff/v4 v4.3.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/cenkalti/backoff/v5 v5.0.2 h1:rIfFVxEf1QsI7E1ZHfp/B4DF/6QBAUhmgkxc0H7Zss8= @@ -85,6 +89,8 @@ github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UF github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudwego/base64x v0.1.4 h1:jwCgWpFanWmN8xoIUHa2rtzmkd5J2plF/dnLS6Xd/0Y= github.com/cloudwego/base64x v0.1.4/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= +github.com/cloudwego/base64x v0.1.5 h1:XPciSp1xaq2VCSt6lF0phncD4koWyULpl5bUxbfCyP4= +github.com/cloudwego/base64x v0.1.5/go.mod h1:0zlkT4Wn5C6NdauXdJRhSKRlJvmclQ1hhJgA0rcu/8w= github.com/cloudwego/iasm v0.2.0 h1:1KNIy1I1H9hNNFEEH3DVnI4UujN+1zjpuk6gwHLTssg= github.com/cloudwego/iasm v0.2.0/go.mod h1:8rXZaNYT2n95jn+zTI1sDr+IgcD2GVs0nlbbQPiEFhY= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= @@ -150,8 +156,14 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.7 h1:SKFKl7kD0RiPdbht0s7hFtjl489WcQ1VyPW8ZzUMYCA= github.com/gabriel-vasile/mimetype v1.4.7/go.mod h1:GDlAgAyIRT27BhFl53XNAFtfjzOkLaF35JdEG0P7LtU= +github.com/gabriel-vasile/mimetype v1.4.8 h1:FfZ3gj38NjllZIeJAmMhr+qKL8Wu+nOoI3GqacKw1NM= +github.com/gabriel-vasile/mimetype v1.4.8/go.mod h1:ByKUIKGjh1ODkGM1asKUbQZOLGrPjydw3hYPU2YU9t8= +github.com/gin-contrib/gzip v1.2.3 h1:dAhT722RuEG330ce2agAs75z7yB+NKvX/ZM1r8w0u2U= +github.com/gin-contrib/gzip v1.2.3/go.mod h1:ad72i4Bzmaypk8M762gNXa2wkxxjbz0icRNnuLJ9a/c= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-contrib/sse v1.0.0 h1:y3bT1mUWUxDpW4JLQg/HnTqV4rozuW4tC9eFKTxYI9E= +github.com/gin-contrib/sse v1.0.0/go.mod h1:zNuFdwarAygJBht0NTKiSi3jRf6RbqeILZ9Sp6Slhe0= github.com/gin-gonic/gin v1.10.0 h1:nTuyha1TYqgedzytsKYqna+DfLos46nTv2ygFy86HFU= github.com/gin-gonic/gin v1.10.0/go.mod h1:4PMNQiOhvDRa013RKVbsiNwoyezlm2rm0uX/T7kzp5Y= github.com/globalsign/mgo v0.0.0-20180905125535-1ca0a4f7cbcb/go.mod h1:xkRDCp4j0OGD1HRkm4kmhM+pmpv3AKq5SU7GMg4oO/Q= @@ -267,6 +279,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.23.0 h1:/PwmTwZhS0dPkav3cdK9kV1FsAmrL8sThn8IHr/sO+o= github.com/go-playground/validator/v10 v10.23.0/go.mod h1:dbuPbCMFw/DrkbEynArYaCwl3amGuJotoKCe95atGMM= +github.com/go-playground/validator/v10 v10.26.0 h1:SP05Nqhjcvz81uJaRfEV0YBSSSGMc/iMaVtFbr3Sw2k= +github.com/go-playground/validator/v10 v10.26.0/go.mod h1:I5QpIEbmr8On7W0TktmJAumgzX4CA1XNl4ZmDuVHKKo= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-sql-driver/mysql v1.6.0 h1:BCTh4TKNUYmOmMUcQ3IipzF5prigylS7XXjEkfCHuOE= github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -300,6 +314,8 @@ github.com/gobuffalo/packr/v2 v2.2.0/go.mod h1:CaAwI0GPIAv+5wKLtv8Afwl+Cm78K/I/V github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= github.com/goccy/go-json v0.10.4 h1:JSwxQzIqKfmFX1swYPpUThQZp/Ka4wzJdK0LWVytLPM= github.com/goccy/go-json v0.10.4/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= +github.com/goccy/go-json v0.10.5 h1:Fq85nIqj+gXn/S5ahsiTlK3TmC85qgirsdTP/+DeaC4= +github.com/goccy/go-json v0.10.5/go.mod h1:oq7eo15ShAhp70Anwd5lgX2pLfOS3QCiwU/PULtXL6M= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= github.com/goccy/go-yaml v1.18.0/go.mod h1:XBurs7gK8ATbW4ZPGKgcbrY1Br56PdM69F7LkFRi1kA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -457,6 +473,8 @@ github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ib github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.9 h1:66ze0taIn2H33fBvCkXuv9BmCwDfafmiIVpKV9kKGuY= github.com/klauspost/cpuid/v2 v2.2.9/go.mod h1:rqkxqrZ1EhYM9G+hXH7YdowN5R5RGN6NK4QwQ3WMXF8= +github.com/klauspost/cpuid/v2 v2.2.10 h1:tBs3QSyvjDyFTq3uoc/9xFpCuOsJQFNPiAhYdw2skhE= +github.com/klauspost/cpuid/v2 v2.2.10/go.mod h1:hqwkgyIinND0mEev00jJYCxPNVRVXFQeu1XKlok6oO0= github.com/knz/go-libedit v1.10.1/go.mod h1:MZTVkCWyz0oBc7JOWP3wNAzd002ZbM/5hgShxwh4x8M= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -785,6 +803,8 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/arch v0.12.0 h1:UsYJhbzPYGsT0HbEdmYcqtCv8UNGvnaL561NnIUvaKg= golang.org/x/arch v0.12.0/go.mod h1:FEVrYAQjsQXMVJ1nsMoVVXPZg6p2JE2mx8psSWTDQys= +golang.org/x/arch v0.15.0 h1:QtOrQd0bTUnhNVNndMpLHNWrDmYzZ2KDqSrEymqInZw= +golang.org/x/arch v0.15.0/go.mod h1:JmwW7aLIoRUKgaTzhkiEFxvcEiQGyOg9BMonBJUS7EE= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190320223903-b7391e95e576/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= diff --git a/pkg/acquisition/acquisition.go b/pkg/acquisition/acquisition.go index 7ae4052b0..7f1fd5ff9 100644 --- a/pkg/acquisition/acquisition.go +++ b/pkg/acquisition/acquisition.go @@ -27,6 +27,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/cwversion/component" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -45,17 +46,17 @@ func (e *DataSourceUnavailableError) Unwrap() error { // The interface each datasource must implement type DataSource interface { - GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module - GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) - UnmarshalConfig(yamlConfig []byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime - Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error // Complete the YAML datasource configuration and perform runtime checks. - ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uniqueID string) error // Configure the datasource - GetMode() string // Get the mode (TAIL, CAT or SERVER) - GetName() string // Get the name of the module - OneShotAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) - StreamingAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error // Start live acquisition (eg, tail a file) - CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) - GetUuid() string // Get the unique identifier of the datasource + GetMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module + GetAggregMetrics() []prometheus.Collector // Returns pointers to metrics that are managed by the module (aggregated mode, limits cardinality) + UnmarshalConfig(yamlConfig []byte) error // Decode and pre-validate the YAML datasource - anything that can be checked before runtime + Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error // Complete the YAML datasource configuration and perform runtime checks. + ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uniqueID string) error // Configure the datasource + GetMode() string // Get the mode (TAIL, CAT or SERVER) + GetName() string // Get the name of the module + OneShotAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error // Start one shot acquisition(eg, cat a file) + StreamingAcquisition(ctx context.Context, out chan types.Event, acquisTomb *tomb.Tomb) error // Start live acquisition (eg, tail a file) + CanRun() error // Whether the datasource can run or not (eg, journalctl on BSD is a non-sense) + GetUuid() string // Get the unique identifier of the datasource Dump() any } @@ -121,7 +122,7 @@ func setupLogger(source, name string, level *log.Level) (*log.Entry, error) { // if the configuration is not valid it returns an error. // If the datasource can't be run (eg. journalctl not available), it still returns an error which // can be checked for the appropriate action. -func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, yamlConfig []byte, metricsLevel int) (DataSource, error) { +func DataSourceConfigure(commonConfig configuration.DataSourceCommonCfg, yamlConfig []byte, metricsLevel metrics.AcquisitionMetricsLevel) (DataSource, error) { dataSrc, err := GetDataSourceIface(commonConfig.Source) if err != nil { return nil, err @@ -179,24 +180,28 @@ func LoadAcquisitionFromDSN(dsn string, labels map[string]string, transformExpr return []DataSource{dataSrc}, nil } -func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) int { +func GetMetricsLevelFromPromCfg(prom *csconfig.PrometheusCfg) metrics.AcquisitionMetricsLevel { if prom == nil { - return configuration.METRICS_FULL + return metrics.AcquisitionMetricsLevelFull } if !prom.Enabled { - return configuration.METRICS_NONE + return metrics.AcquisitionMetricsLevelNone } - if prom.Level == configuration.CFG_METRICS_AGGREGATE { - return configuration.METRICS_AGGREGATE + if prom.Level == metrics.MetricsLevelNone { + return metrics.AcquisitionMetricsLevelNone } - if prom.Level == configuration.CFG_METRICS_FULL { - return configuration.METRICS_FULL + if prom.Level == metrics.MetricsLevelAggregated { + return metrics.AcquisitionMetricsLevelAggregated } - return configuration.METRICS_FULL + if prom.Level == metrics.MetricsLevelFull { + return metrics.AcquisitionMetricsLevelFull + } + + return metrics.AcquisitionMetricsLevelFull } func detectTypes(r io.Reader) ([]string, error) { @@ -230,7 +235,7 @@ func detectTypes(r io.Reader) ([]string, error) { } // sourcesFromFile reads and parses one acquisition file into DataSources. -func sourcesFromFile(acquisFile string, metrics_level int) ([]DataSource, error) { +func sourcesFromFile(acquisFile string, metrics_level metrics.AcquisitionMetricsLevel) ([]DataSource, error) { var sources []DataSource log.Infof("loading acquisition file : %s", acquisFile) diff --git a/pkg/acquisition/acquisition_test.go b/pkg/acquisition/acquisition_test.go index a21c01a77..d75055faf 100644 --- a/pkg/acquisition/acquisition_test.go +++ b/pkg/acquisition/acquisition_test.go @@ -19,6 +19,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/csconfig" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -37,7 +38,7 @@ func (f *MockSource) UnmarshalConfig(cfg []byte) error { return nil } -func (f *MockSource) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error { +func (f *MockSource) Configure(cfg []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { f.logger = logger if err := f.UnmarshalConfig(cfg); err != nil { return err @@ -191,7 +192,7 @@ filename: foo.log common := configuration.DataSourceCommonCfg{} err := yaml.Unmarshal([]byte(tc.String), &common) require.NoError(t, err) - ds, err := DataSourceConfigure(common, []byte(tc.String), configuration.METRICS_NONE) + ds, err := DataSourceConfigure(common, []byte(tc.String), metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.ExpectedError) if tc.ExpectedError != "" { @@ -331,7 +332,7 @@ type MockCat struct { logger *log.Entry } -func (f *MockCat) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error { +func (f *MockCat) Configure(cfg []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { f.logger = logger if f.Mode == "" { f.Mode = configuration.CAT_MODE @@ -376,7 +377,7 @@ type MockTail struct { logger *log.Entry } -func (f *MockTail) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error { +func (f *MockTail) Configure(cfg []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { f.logger = logger if f.Mode == "" { f.Mode = configuration.TAIL_MODE @@ -531,7 +532,7 @@ type MockSourceByDSN struct { } func (f *MockSourceByDSN) UnmarshalConfig(cfg []byte) error { return nil } -func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry, metricsLevel int) error { +func (f *MockSourceByDSN) Configure(cfg []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { return nil } func (f *MockSourceByDSN) GetMode() string { return f.Mode } diff --git a/pkg/acquisition/configuration/configuration.go b/pkg/acquisition/configuration/configuration.go index 5a277cdf3..e669a975e 100644 --- a/pkg/acquisition/configuration/configuration.go +++ b/pkg/acquisition/configuration/configuration.go @@ -20,14 +20,3 @@ const ( CAT_MODE = "cat" SERVER_MODE = "server" // No difference with tail, just a bit more verbose ) - -const ( - METRICS_NONE = iota - METRICS_AGGREGATE - METRICS_FULL -) - -const ( - CFG_METRICS_AGGREGATE = "aggregated" - CFG_METRICS_FULL = "full" -) diff --git a/pkg/acquisition/modules/appsec/appsec.go b/pkg/acquisition/modules/appsec/appsec.go index 0566e3965..d440b99a9 100644 --- a/pkg/acquisition/modules/appsec/appsec.go +++ b/pkg/acquisition/modules/appsec/appsec.go @@ -29,6 +29,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists" "github.com/crowdsecurity/crowdsec/pkg/csconfig" "github.com/crowdsecurity/crowdsec/pkg/csnet" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -61,7 +62,7 @@ type AppsecSourceConfig struct { // runtime structure of AppsecSourceConfig type AppsecSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config AppsecSourceConfig logger *log.Entry mux *http.ServeMux @@ -170,11 +171,13 @@ func (w *AppsecSource) UnmarshalConfig(yamlConfig []byte) error { } func (w *AppsecSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} + return []prometheus.Collector{metrics.AppsecReqCounter, metrics.AppsecBlockCounter, metrics.AppsecRuleHits, + metrics.AppsecOutbandParsingHistogram, metrics.AppsecInbandParsingHistogram, metrics.AppsecGlobalParsingHistogram} } func (w *AppsecSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{AppsecReqCounter, AppsecBlockCounter, AppsecRuleHits, AppsecOutbandParsingHistogram, AppsecInbandParsingHistogram, AppsecGlobalParsingHistogram} + return []prometheus.Collector{metrics.AppsecReqCounter, metrics.AppsecBlockCounter, metrics.AppsecRuleHits, + metrics.AppsecOutbandParsingHistogram, metrics.AppsecInbandParsingHistogram, metrics.AppsecGlobalParsingHistogram} } func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, error) { @@ -199,7 +202,7 @@ func loadCertPool(caCertPath string, logger log.FieldLogger) (*x509.CertPool, er return caCertPool, nil } -func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (w *AppsecSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { err := w.UnmarshalConfig(yamlConfig) if err != nil { return fmt.Errorf("unable to parse appsec configuration: %w", err) @@ -572,7 +575,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { "client_ip": parsedRequest.ClientIP, }) - AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() + metrics.AppsecReqCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() w.InChan <- parsedRequest @@ -583,7 +586,7 @@ func (w *AppsecSource) appsecHandler(rw http.ResponseWriter, r *http.Request) { response := <-parsedRequest.ResponseChannel if response.InBandInterrupt { - AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() + metrics.AppsecBlockCounter.With(prometheus.Labels{"source": parsedRequest.RemoteAddrNormalized, "appsec_engine": parsedRequest.AppsecEngine}).Inc() } statusCode, appsecResponse := w.AppsecRuntime.GenerateResponse(response, logger) diff --git a/pkg/acquisition/modules/appsec/appsec_runner.go b/pkg/acquisition/modules/appsec/appsec_runner.go index 1d37781ae..ddb768f8c 100644 --- a/pkg/acquisition/modules/appsec/appsec_runner.go +++ b/pkg/acquisition/modules/appsec/appsec_runner.go @@ -18,6 +18,7 @@ import ( _ "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/appsec/bodyprocessors" "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/appsec/allowlists" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -364,7 +365,7 @@ func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { // time spent to process in band rules inBandParsingElapsed := time.Since(startInBandParsing) - AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(inBandParsingElapsed.Seconds()) + metrics.AppsecInbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(inBandParsingElapsed.Seconds()) if request.Tx.IsInterrupted() { r.handleInBandInterrupt(request) @@ -403,7 +404,7 @@ func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { // time spent to process out of band rules outOfBandParsingElapsed := time.Since(startOutOfBandParsing) - AppsecOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(outOfBandParsingElapsed.Seconds()) + metrics.AppsecOutbandParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(outOfBandParsingElapsed.Seconds()) if request.Tx.IsInterrupted() { r.handleOutBandInterrupt(request) } @@ -414,7 +415,7 @@ func (r *AppsecRunner) handleRequest(request *appsec.ParsedRequest) { } // time spent to process inband AND out of band rules globalParsingElapsed := time.Since(startGlobalParsing) - AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds()) + metrics.AppsecGlobalParsingHistogram.With(prometheus.Labels{"source": request.RemoteAddrNormalized, "appsec_engine": request.AppsecEngine}).Observe(globalParsingElapsed.Seconds()) } func (r *AppsecRunner) Run(t *tomb.Tomb) error { diff --git a/pkg/acquisition/modules/appsec/metrics.go b/pkg/acquisition/modules/appsec/metrics.go index 132759338..0a9fa1bf0 100644 --- a/pkg/acquisition/modules/appsec/metrics.go +++ b/pkg/acquisition/modules/appsec/metrics.go @@ -1,54 +1 @@ package appsecacquisition - -import "github.com/prometheus/client_golang/prometheus" - -var AppsecGlobalParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the Application Security Engine.", - Name: "cs_appsec_parsing_time_seconds", - Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, - }, - []string{"source", "appsec_engine"}, -) - -var AppsecInbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the inband Application Security Engine.", - Name: "cs_appsec_inband_parsing_time_seconds", - Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, - }, - []string{"source", "appsec_engine"}, -) - -var AppsecOutbandParsingHistogram = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Help: "Time spent processing a request by the Application Security Engine.", - Name: "cs_appsec_outband_parsing_time_seconds", - Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, - }, - []string{"source", "appsec_engine"}, -) - -var AppsecReqCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_appsec_reqs_total", - Help: "Total events processed by the Application Security Engine.", - }, - []string{"source", "appsec_engine"}, -) - -var AppsecBlockCounter = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_appsec_block_total", - Help: "Total events blocked by the Application Security Engine.", - }, - []string{"source", "appsec_engine"}, -) - -var AppsecRuleHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_appsec_rule_hits", - Help: "Count of triggered rule, by rule_name, type (inband/outofband), appsec_engine and source", - }, - []string{"rule_name", "type", "appsec_engine", "source"}, -) diff --git a/pkg/acquisition/modules/appsec/utils.go b/pkg/acquisition/modules/appsec/utils.go index bde837e69..a2675b4e0 100644 --- a/pkg/acquisition/modules/appsec/utils.go +++ b/pkg/acquisition/modules/appsec/utils.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/alertcontext" "github.com/crowdsecurity/crowdsec/pkg/appsec" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/models" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -263,7 +264,7 @@ func (r *AppsecRunner) AccumulateTxToEvent(evt *types.Event, req *appsec.ParsedR name = fmt.Sprintf("native_rule:%d", rule.Rule().ID()) } - AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc() + metrics.AppsecRuleHits.With(prometheus.Labels{"rule_name": ruleNameProm, "type": kind, "source": req.RemoteAddrNormalized, "appsec_engine": req.AppsecEngine}).Inc() matchedZones := make([]string, 0) diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch.go b/pkg/acquisition/modules/cloudwatch/cloudwatch.go index 8d5fdb1e5..7536fbe4f 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch.go @@ -21,31 +21,16 @@ import ( "gopkg.in/tomb.v2" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/parser" "github.com/crowdsecurity/crowdsec/pkg/types" ) -var openedStreams = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_cloudwatch_openstreams_total", - Help: "Number of opened stream within group.", - }, - []string{"group"}, -) - var streamIndexMutex = sync.Mutex{} -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_cloudwatch_stream_hits_total", - Help: "Number of event read from stream.", - }, - []string{"group", "stream"}, -) - // CloudwatchSource is the runtime instance keeping track of N streams within 1 cloudwatch group type CloudwatchSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config CloudwatchSourceConfiguration /*runtime stuff*/ logger *log.Entry @@ -155,7 +140,7 @@ func (cw *CloudwatchSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (cw *CloudwatchSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { err := cw.UnmarshalConfig(yamlConfig) if err != nil { return err @@ -267,11 +252,11 @@ func (cw *CloudwatchSource) StreamingAcquisition(ctx context.Context, out chan t } func (cw *CloudwatchSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, openedStreams} + return []prometheus.Collector{metrics.CloudWatchDatasourceLinesRead, metrics.CloudWatchDatasourceOpenedStreams} } func (cw *CloudwatchSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, openedStreams} + return []prometheus.Collector{metrics.CloudWatchDatasourceLinesRead, metrics.CloudWatchDatasourceOpenedStreams} } func (cw *CloudwatchSource) GetMode() string { @@ -414,8 +399,8 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr cw.logger.Debugf("stream %s already exists, but is dead", newStream.StreamName) cw.monitoredStreams = slices.Delete(cw.monitoredStreams, idx, idx+1) - if cw.metricsLevel != configuration.METRICS_NONE { - openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Dec() + if cw.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.CloudWatchDatasourceOpenedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Dec() } break @@ -429,8 +414,8 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr // let's start watching this stream if shouldCreate { - if cw.metricsLevel != configuration.METRICS_NONE { - openedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Inc() + if cw.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.CloudWatchDatasourceOpenedStreams.With(prometheus.Labels{"group": newStream.GroupName}).Inc() } newStream.t = tomb.Tomb{} @@ -449,8 +434,8 @@ func (cw *CloudwatchSource) LogStreamManager(ctx context.Context, in chan LogStr if !cw.monitoredStreams[idx].t.Alive() { cw.logger.Debugf("remove dead stream %s", stream.StreamName) - if cw.metricsLevel != configuration.METRICS_NONE { - openedStreams.With(prometheus.Labels{"group": cw.monitoredStreams[idx].GroupName}).Dec() + if cw.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.CloudWatchDatasourceOpenedStreams.With(prometheus.Labels{"group": cw.monitoredStreams[idx].GroupName}).Dec() } } else { newMonitoredStreams = append(newMonitoredStreams, stream) @@ -541,8 +526,13 @@ func (cw *CloudwatchSource) TailLogStream(ctx context.Context, cfg *LogStreamTai } else { cfg.logger.Debugf("pushing message : %s", evt.Line.Raw) - if cw.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"group": cfg.GroupName, "stream": cfg.StreamName}).Inc() + if cw.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.CloudWatchDatasourceLinesRead.With( + prometheus.Labels{"group": cfg.GroupName, + "stream": cfg.StreamName, + "datasource_type": "cloudwatch", + "acquis_type": evt.Line.Labels["type"], + }).Inc() } outChan <- evt } diff --git a/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go b/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go index 4ce4e718f..906658e70 100644 --- a/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go +++ b/pkg/acquisition/modules/cloudwatch/cloudwatch_test.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -325,7 +325,7 @@ stream_name: test_stream`, dbgLogger.Infof("starting test") cw := CloudwatchSource{} - err := cw.Configure([]byte(tc.config), dbgLogger, configuration.METRICS_NONE) + err := cw.Configure([]byte(tc.config), dbgLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(s.T(), err, tc.expectedCfgErr) if tc.expectedCfgErr != "" { @@ -442,7 +442,7 @@ stream_name: test_stream`, dbgLogger.Logger.SetLevel(logrus.DebugLevel) cw := CloudwatchSource{} - err := cw.Configure([]byte(tc.config), dbgLogger, configuration.METRICS_NONE) + err := cw.Configure([]byte(tc.config), dbgLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(s.T(), err, tc.expectedCfgErr) if tc.expectedCfgErr != "" { diff --git a/pkg/acquisition/modules/docker/docker.go b/pkg/acquisition/modules/docker/docker.go index f48cb9365..21a9ee1f2 100644 --- a/pkg/acquisition/modules/docker/docker.go +++ b/pkg/acquisition/modules/docker/docker.go @@ -26,14 +26,9 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/types" -) -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_dockersource_hits_total", - Help: "Total lines that were read.", - }, - []string{"source"}) + "github.com/crowdsecurity/crowdsec/pkg/metrics" +) type DockerConfiguration struct { configuration.DataSourceCommonCfg `yaml:",inline"` @@ -52,7 +47,7 @@ type DockerConfiguration struct { } type DockerSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config DockerConfiguration runningContainerState map[string]*ContainerConfig compiledContainerName []*regexp.Regexp @@ -137,7 +132,7 @@ func (d *DockerSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (d *DockerSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { d.logger = logger d.metricsLevel = metricsLevel @@ -343,8 +338,8 @@ func (d *DockerSource) OneShotAcquisition(ctx context.Context, out chan types.Ev l.Process = true l.Module = d.GetName() - if d.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": containerConfig.Name}).Inc() + if d.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.DockerDatasourceLinesRead.With(prometheus.Labels{"source": containerConfig.Name, "acquis_type": l.Labels["type"], "datasource_type": "docker"}).Inc() } evt := types.MakeEvent(true, types.LOG, true) @@ -377,11 +372,11 @@ func (d *DockerSource) OneShotAcquisition(ctx context.Context, out chan types.Ev } func (d *DockerSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.DockerDatasourceLinesRead} } func (d *DockerSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.DockerDatasourceLinesRead} } func (d *DockerSource) GetName() string { @@ -729,7 +724,7 @@ func (d *DockerSource) TailDocker(ctx context.Context, container *ContainerConfi l.Module = d.GetName() evt := types.MakeEvent(d.Config.UseTimeMachine, types.LOG, true) evt.Line = l - linesRead.With(prometheus.Labels{"source": container.Name}).Inc() + metrics.DockerDatasourceLinesRead.With(prometheus.Labels{"source": container.Name, "datasource_type": "docker", "acquis_type": evt.Line.Labels["type"]}).Inc() outChan <- evt d.logger.Debugf("Sent line to parsing: %+v", evt.Line.Raw) case <-readerTomb.Dying(): diff --git a/pkg/acquisition/modules/docker/docker_test.go b/pkg/acquisition/modules/docker/docker_test.go index b1e793100..52c0b82c4 100644 --- a/pkg/acquisition/modules/docker/docker_test.go +++ b/pkg/acquisition/modules/docker/docker_test.go @@ -22,7 +22,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -71,7 +71,7 @@ container_name: for _, tc := range tests { t.Run(tc.config, func(t *testing.T) { f := DockerSource{} - err := f.Configure([]byte(tc.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.expectedErr) }) } @@ -197,7 +197,7 @@ container_name_regexp: out := make(chan types.Event) dockerSource := DockerSource{} - err := dockerSource.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := dockerSource.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) dockerSource.Client = new(mockDockerCli) diff --git a/pkg/acquisition/modules/file/file.go b/pkg/acquisition/modules/file/file.go index 935579506..0da9644dc 100644 --- a/pkg/acquisition/modules/file/file.go +++ b/pkg/acquisition/modules/file/file.go @@ -27,18 +27,12 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) const defaultPollInterval = 30 * time.Second -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_filesource_hits_total", - Help: "Total lines that were read.", - }, - []string{"source"}) - type FileConfiguration struct { Filenames []string ExcludeRegexps []string `yaml:"exclude_regexps"` @@ -52,7 +46,7 @@ type FileConfiguration struct { } type FileSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config FileConfiguration watcher *fsnotify.Watcher watchedDirectories map[string]bool @@ -107,7 +101,7 @@ func (f *FileSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (f *FileSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { f.logger = logger f.metricsLevel = metricsLevel @@ -298,11 +292,11 @@ func (f *FileSource) OneShotAcquisition(ctx context.Context, out chan types.Even } func (f *FileSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.FileDatasourceLinesRead} } func (f *FileSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.FileDatasourceLinesRead} } func (f *FileSource) GetName() string { @@ -578,12 +572,12 @@ func (f *FileSource) tailFile(out chan types.Event, t *tomb.Tomb, tail *tail.Tai continue } - if f.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": tail.Filename}).Inc() + if f.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.FileDatasourceLinesRead.With(prometheus.Labels{"source": tail.Filename, "datasource_type": "file", "acquis_type": f.config.Labels["type"]}).Inc() } src := tail.Filename - if f.metricsLevel == configuration.METRICS_AGGREGATE { + if f.metricsLevel == metrics.AcquisitionMetricsLevelAggregated { src = filepath.Base(tail.Filename) } @@ -656,7 +650,7 @@ func (f *FileSource) readFile(filename string, out chan types.Event, t *tomb.Tom Module: f.GetName(), } logger.Debugf("line %s", l.Raw) - linesRead.With(prometheus.Labels{"source": filename}).Inc() + metrics.FileDatasourceLinesRead.With(prometheus.Labels{"source": filename, "datasource_type": "file", "acquis_type": l.Labels["type"]}).Inc() // we're reading logs at once, it must be time-machine buckets out <- types.Event{Line: l, Process: true, Type: types.LOG, ExpectMode: types.TIMEMACHINE, Unmarshaled: make(map[string]any)} diff --git a/pkg/acquisition/modules/file/file_test.go b/pkg/acquisition/modules/file/file_test.go index e5f260c01..1ddb8109f 100644 --- a/pkg/acquisition/modules/file/file_test.go +++ b/pkg/acquisition/modules/file/file_test.go @@ -16,8 +16,8 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" fileacquisition "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/file" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -61,7 +61,7 @@ filenames: ["ase.log"]`, for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := fileacquisition.FileSource{} - err := f.Configure([]byte(tc.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.expectedErr) }) } @@ -225,7 +225,7 @@ filename: %s`, deletedFile), tc.setup() } - err := f.Configure([]byte(tc.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.expectedConfigErr) if tc.expectedConfigErr != "" { @@ -393,7 +393,7 @@ force_inotify: true`, testPattern), tc.setup() } - err := f.Configure([]byte(tc.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) if tc.afterConfigure != nil { @@ -491,7 +491,7 @@ exclude_regexps: ["\\.gz$"]` subLogger := logger.WithField("type", "file") f := fileacquisition.FileSource{} - err := f.Configure([]byte(config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) require.NotNil(t, hook.LastEntry()) @@ -543,7 +543,7 @@ mode: tail for _, tc := range tests { t.Run(tc.name, func(t *testing.T) { f := &fileacquisition.FileSource{} - err := f.Configure([]byte(tc.config), log.NewEntry(log.New()), configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), log.NewEntry(log.New()), metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.wantErr) }) } @@ -567,7 +567,7 @@ mode: tail config := []byte(yamlConfig) f := &fileacquisition.FileSource{} - err := f.Configure(config, log.NewEntry(log.New()), configuration.METRICS_NONE) + err := f.Configure(config, log.NewEntry(log.New()), metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) // Create channel for events @@ -619,7 +619,7 @@ mode: tail config := []byte(yamlConfig) f := &fileacquisition.FileSource{} - err = f.Configure(config, log.NewEntry(log.New()), configuration.METRICS_NONE) + err = f.Configure(config, log.NewEntry(log.New()), metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) eventChan := make(chan types.Event) diff --git a/pkg/acquisition/modules/http/http.go b/pkg/acquisition/modules/http/http.go index b6f0b0638..30137e41e 100644 --- a/pkg/acquisition/modules/http/http.go +++ b/pkg/acquisition/modules/http/http.go @@ -24,17 +24,11 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/csnet" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) -var dataSourceName = "http" - -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_httpsource_hits_total", - Help: "Total lines that were read from http source", - }, - []string{"path", "src"}) +const dataSourceName = "http" type HttpConfiguration struct { // IPFilter []string `yaml:"ip_filter"` @@ -66,7 +60,7 @@ type TLSConfig struct { } type HTTPSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config HttpConfiguration logger *log.Entry Server *http.Server @@ -160,7 +154,7 @@ func (hc *HttpConfiguration) Validate() error { return nil } -func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (h *HTTPSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { h.logger = logger h.metricsLevel = metricsLevel @@ -197,11 +191,11 @@ func (h *HTTPSource) CanRun() error { } func (h *HTTPSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.HTTPDataSourceLinesRead} } func (h *HTTPSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.HTTPDataSourceLinesRead} } func (h *HTTPSource) Dump() interface{} { @@ -326,17 +320,20 @@ func (h *HTTPSource) processRequest(w http.ResponseWriter, r *http.Request, hc * Module: h.GetName(), } - if h.metricsLevel == configuration.METRICS_AGGREGATE { + if h.metricsLevel == metrics.AcquisitionMetricsLevelAggregated { line.Src = hc.Path } evt := types.MakeEvent(h.Config.UseTimeMachine, types.LOG, true) evt.Line = line - if h.metricsLevel == configuration.METRICS_AGGREGATE { - linesRead.With(prometheus.Labels{"path": hc.Path, "src": ""}).Inc() - } else if h.metricsLevel == configuration.METRICS_FULL { - linesRead.With(prometheus.Labels{"path": hc.Path, "src": srcHost}).Inc() + switch h.metricsLevel { + case metrics.AcquisitionMetricsLevelAggregated: + metrics.HTTPDataSourceLinesRead.With(prometheus.Labels{"path": hc.Path, "src": "", "datasource_type": "http", "acquis_type": hc.Labels["type"]}).Inc() + case metrics.AcquisitionMetricsLevelFull: + metrics.HTTPDataSourceLinesRead.With(prometheus.Labels{"path": hc.Path, "src": srcHost, "datasource_type": "http", "acquis_type": hc.Labels["type"]}).Inc() + case metrics.AcquisitionMetricsLevelNone: + // No metrics for this level } h.logger.Tracef("line to send: %+v", line) diff --git a/pkg/acquisition/modules/http/http_test.go b/pkg/acquisition/modules/http/http_test.go index 4badccafd..72faaaf8f 100644 --- a/pkg/acquisition/modules/http/http_test.go +++ b/pkg/acquisition/modules/http/http_test.go @@ -24,6 +24,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -222,7 +223,7 @@ func TestGetName(t *testing.T) { assert.Equal(t, "http", h.GetName()) } -func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel int) (chan types.Event, *prometheus.Registry, *tomb.Tomb) { +func SetupAndRunHTTPSource(t *testing.T, h *HTTPSource, config []byte, metricLevel metrics.AcquisitionMetricsLevel) (chan types.Event, *prometheus.Registry, *tomb.Tomb) { ctx := t.Context() subLogger := log.WithFields(log.Fields{ "type": "http", @@ -894,11 +895,15 @@ func assertMetrics(t *testing.T, reg *prometheus.Registry, metrics []prometheus. for _, metric := range metricFamily.GetMetric() { assert.InDelta(t, float64(expected), metric.GetCounter().GetValue(), 0.000001) labels := metric.GetLabel() - assert.Len(t, labels, 2) - assert.Equal(t, "path", labels[0].GetName()) - assert.Equal(t, "/test", labels[0].GetValue()) - assert.Equal(t, "src", labels[1].GetName()) - assert.Equal(t, "127.0.0.1", labels[1].GetValue()) + assert.Len(t, labels, 4) + assert.Equal(t, "acquis_type", labels[0].GetName()) + assert.Empty(t, labels[0].GetValue()) + assert.Equal(t, "datasource_type", labels[1].GetName()) + assert.Equal(t, "http", labels[1].GetValue()) + assert.Equal(t, "path", labels[2].GetName()) + assert.Equal(t, "/test", labels[2].GetValue()) + assert.Equal(t, "src", labels[3].GetName()) + assert.Equal(t, "127.0.0.1", labels[3].GetValue()) } } } diff --git a/pkg/acquisition/modules/journalctl/journalctl.go b/pkg/acquisition/modules/journalctl/journalctl.go index 5bad415e9..a4a1bf9ae 100644 --- a/pkg/acquisition/modules/journalctl/journalctl.go +++ b/pkg/acquisition/modules/journalctl/journalctl.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -27,7 +28,7 @@ type JournalCtlConfiguration struct { } type JournalCtlSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config JournalCtlConfiguration logger *log.Entry src string @@ -41,13 +42,6 @@ var ( journalctlArgstreaming = []string{"--follow", "-n", "0"} ) -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_journalctlsource_hits_total", - Help: "Total lines that were read.", - }, - []string{"source"}) - func readLine(scanner *bufio.Scanner, out chan string, errChan chan error) error { for scanner.Scan() { txt := scanner.Text() @@ -143,8 +137,8 @@ func (j *JournalCtlSource) runJournalCtl(ctx context.Context, out chan types.Eve l.Process = true l.Module = j.GetName() - if j.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": j.src}).Inc() + if j.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.JournalCtlDataSourceLinesRead.With(prometheus.Labels{"source": j.src, "datasource_type": "journalctl", "acquis_type": l.Labels["type"]}).Inc() } evt := types.MakeEvent(j.config.UseTimeMachine, types.LOG, true) @@ -172,11 +166,11 @@ func (j *JournalCtlSource) GetUuid() string { } func (j *JournalCtlSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.JournalCtlDataSourceLinesRead} } func (j *JournalCtlSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.JournalCtlDataSourceLinesRead} } func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error { @@ -210,7 +204,7 @@ func (j *JournalCtlSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (j *JournalCtlSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { j.logger = logger j.metricsLevel = metricsLevel diff --git a/pkg/acquisition/modules/journalctl/journalctl_test.go b/pkg/acquisition/modules/journalctl/journalctl_test.go index 7ab632b7c..97d3d4658 100644 --- a/pkg/acquisition/modules/journalctl/journalctl_test.go +++ b/pkg/acquisition/modules/journalctl/journalctl_test.go @@ -15,7 +15,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -51,7 +51,7 @@ journalctl_filter: for _, tc := range tests { t.Run(tc.config, func(t *testing.T) { f := JournalCtlSource{} - err := f.Configure([]byte(tc.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(tc.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, tc.expectedErr) }) } @@ -157,7 +157,7 @@ journalctl_filter: out := make(chan types.Event, 100) j := JournalCtlSource{} - err := j.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := j.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } @@ -227,7 +227,7 @@ journalctl_filter: out := make(chan types.Event) j := JournalCtlSource{} - err := j.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := j.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } diff --git a/pkg/acquisition/modules/kafka/kafka.go b/pkg/acquisition/modules/kafka/kafka.go index 3109a242d..53b4c9f2e 100644 --- a/pkg/acquisition/modules/kafka/kafka.go +++ b/pkg/acquisition/modules/kafka/kafka.go @@ -20,18 +20,12 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) const dataSourceName = "kafka" -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_kafkasource_hits_total", - Help: "Total lines that were read from topic", - }, - []string{"topic"}) - type KafkaConfiguration struct { Brokers []string `yaml:"brokers"` Topic string `yaml:"topic"` @@ -59,7 +53,7 @@ type KafkaBatchConfiguration struct { } type KafkaSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config KafkaConfiguration logger *log.Entry Reader *kafka.Reader @@ -94,7 +88,7 @@ func (k *KafkaSource) UnmarshalConfig(yamlConfig []byte) error { return err } -func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (k *KafkaSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { k.logger = logger k.metricsLevel = metricsLevel @@ -145,11 +139,11 @@ func (*KafkaSource) CanRun() error { } func (*KafkaSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.KafkaDataSourceLinesRead} } func (*KafkaSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.KafkaDataSourceLinesRead} } func (k *KafkaSource) Dump() any { @@ -189,8 +183,8 @@ func (k *KafkaSource) ReadMessage(ctx context.Context, out chan types.Event) err } k.logger.Tracef("line with message read from topic '%s': %+v", k.Config.Topic, l) - if k.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"topic": k.Config.Topic}).Inc() + if k.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.KafkaDataSourceLinesRead.With(prometheus.Labels{"topic": k.Config.Topic, "datasource_type": "kafka", "acquis_type": l.Labels["type"]}).Inc() } evt := types.MakeEvent(k.Config.UseTimeMachine, types.LOG, true) diff --git a/pkg/acquisition/modules/kafka/kafka_test.go b/pkg/acquisition/modules/kafka/kafka_test.go index 758acabef..6d2514476 100644 --- a/pkg/acquisition/modules/kafka/kafka_test.go +++ b/pkg/acquisition/modules/kafka/kafka_test.go @@ -14,7 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -74,7 +74,7 @@ group_id: crowdsec`, for _, test := range tests { k := KafkaSource{} - err := k.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := k.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.AssertErrorContains(t, err, test.expectedErr) } } @@ -168,7 +168,7 @@ func TestStreamingAcquisition(t *testing.T) { source: kafka brokers: - localhost:9092 -topic: crowdsecplaintext`), subLogger, configuration.METRICS_NONE) +topic: crowdsecplaintext`), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("could not configure kafka source : %s", err) } @@ -245,7 +245,7 @@ tls: client_cert: ./testdata/kafkaClient.certificate.pem client_key: ./testdata/kafkaClient.key ca_cert: ./testdata/snakeoil-ca-1.crt - `), subLogger, configuration.METRICS_NONE) + `), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("could not configure kafka source : %s", err) } diff --git a/pkg/acquisition/modules/kinesis/kinesis.go b/pkg/acquisition/modules/kinesis/kinesis.go index 8d6439217..f92e815b4 100644 --- a/pkg/acquisition/modules/kinesis/kinesis.go +++ b/pkg/acquisition/modules/kinesis/kinesis.go @@ -23,6 +23,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -40,7 +41,7 @@ type KinesisConfiguration struct { } type KinesisSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config KinesisConfiguration logger *log.Entry kClient *kinesis.Kinesis @@ -62,22 +63,6 @@ type CloudwatchSubscriptionLogEvent struct { Timestamp int64 `json:"timestamp"` } -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_kinesis_stream_hits_total", - Help: "Number of event read per stream.", - }, - []string{"stream"}, -) - -var linesReadShards = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_kinesis_shards_hits_total", - Help: "Number of event read per shards.", - }, - []string{"stream", "shard"}, -) - func (k *KinesisSource) GetUuid() string { return k.Config.UniqueId } @@ -119,11 +104,11 @@ func (k *KinesisSource) newClient() error { } func (k *KinesisSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, linesReadShards} + return []prometheus.Collector{metrics.KinesisDataSourceLinesRead, metrics.KinesisDataSourceLinesReadShards} } func (k *KinesisSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, linesReadShards} + return []prometheus.Collector{metrics.KinesisDataSourceLinesRead, metrics.KinesisDataSourceLinesReadShards} } func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { @@ -161,7 +146,7 @@ func (k *KinesisSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (k *KinesisSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { k.logger = logger k.metricsLevel = metricsLevel @@ -314,14 +299,14 @@ func (k *KinesisSource) RegisterConsumer() (*kinesis.RegisterStreamConsumerOutpu func (k *KinesisSource) ParseAndPushRecords(records []*kinesis.Record, out chan types.Event, logger *log.Entry, shardId string) { for _, record := range records { if k.Config.StreamARN != "" { - if k.metricsLevel != configuration.METRICS_NONE { - linesReadShards.With(prometheus.Labels{"stream": k.Config.StreamARN, "shard": shardId}).Inc() - linesRead.With(prometheus.Labels{"stream": k.Config.StreamARN}).Inc() + if k.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.KinesisDataSourceLinesReadShards.With(prometheus.Labels{"stream": k.Config.StreamARN, "shard": shardId}).Inc() + metrics.KinesisDataSourceLinesRead.With(prometheus.Labels{"stream": k.Config.StreamARN, "datasource_type": "kinesis", "acquis_type": k.Config.Labels["type"]}).Inc() } } else { - if k.metricsLevel != configuration.METRICS_NONE { - linesReadShards.With(prometheus.Labels{"stream": k.Config.StreamName, "shard": shardId}).Inc() - linesRead.With(prometheus.Labels{"stream": k.Config.StreamName}).Inc() + if k.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.KinesisDataSourceLinesReadShards.With(prometheus.Labels{"stream": k.Config.StreamName, "shard": shardId}).Inc() + metrics.KinesisDataSourceLinesRead.With(prometheus.Labels{"stream": k.Config.StreamName, "datasource_type": "kinesis", "acquis_type": k.Config.Labels["type"]}).Inc() } } diff --git a/pkg/acquisition/modules/kinesis/kinesis_test.go b/pkg/acquisition/modules/kinesis/kinesis_test.go index 3373bba74..91615a603 100644 --- a/pkg/acquisition/modules/kinesis/kinesis_test.go +++ b/pkg/acquisition/modules/kinesis/kinesis_test.go @@ -20,7 +20,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -134,7 +134,7 @@ stream_arn: arn:aws:kinesis:eu-west-1:123456789012:stream/my-stream`, for _, test := range tests { t.Run(test.config, func(t *testing.T) { f := KinesisSource{} - err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.AssertErrorContains(t, err, test.expectedErr) }) } @@ -162,7 +162,7 @@ stream_name: stream-1-shard`, for _, test := range tests { f := KinesisSource{} config := fmt.Sprintf(test.config, endpoint) - err := f.Configure([]byte(config), log.WithField("type", "kinesis"), configuration.METRICS_NONE) + err := f.Configure([]byte(config), log.WithField("type", "kinesis"), metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) tomb := &tomb.Tomb{} @@ -207,7 +207,7 @@ stream_name: stream-2-shards`, for _, test := range tests { f := KinesisSource{} config := fmt.Sprintf(test.config, endpoint) - err := f.Configure([]byte(config), log.WithField("type", "kinesis"), configuration.METRICS_NONE) + err := f.Configure([]byte(config), log.WithField("type", "kinesis"), metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) tomb := &tomb.Tomb{} @@ -256,7 +256,7 @@ from_subscription: true`, for _, test := range tests { f := KinesisSource{} config := fmt.Sprintf(test.config, endpoint) - err := f.Configure([]byte(config), log.WithField("type", "kinesis"), configuration.METRICS_NONE) + err := f.Configure([]byte(config), log.WithField("type", "kinesis"), metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) tomb := &tomb.Tomb{} diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go index 344c4fd38..22c737f0e 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit.go @@ -18,6 +18,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -29,7 +30,7 @@ type KubernetesAuditConfiguration struct { } type KubernetesAuditSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config KubernetesAuditConfiguration logger *log.Entry mux *http.ServeMux @@ -38,30 +39,16 @@ type KubernetesAuditSource struct { addr string } -var eventCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_k8sauditsource_hits_total", - Help: "Total number of events received by k8s-audit source", - }, - []string{"source"}) - -var requestCount = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_k8sauditsource_requests_total", - Help: "Total number of requests received", - }, - []string{"source"}) - func (ka *KubernetesAuditSource) GetUuid() string { return ka.config.UniqueId } func (ka *KubernetesAuditSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{eventCount, requestCount} + return []prometheus.Collector{metrics.K8SAuditDataSourceEventCount, metrics.K8SAuditDataSourceRequestCount} } func (ka *KubernetesAuditSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{eventCount, requestCount} + return []prometheus.Collector{metrics.K8SAuditDataSourceEventCount, metrics.K8SAuditDataSourceRequestCount} } func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { @@ -97,7 +84,7 @@ func (ka *KubernetesAuditSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { +func (ka *KubernetesAuditSource) Configure(config []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { ka.logger = logger ka.metricsLevel = metricsLevel @@ -179,8 +166,8 @@ func (ka *KubernetesAuditSource) Dump() interface{} { } func (ka *KubernetesAuditSource) webhookHandler(w http.ResponseWriter, r *http.Request) { - if ka.metricsLevel != configuration.METRICS_NONE { - requestCount.WithLabelValues(ka.addr).Inc() + if ka.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.K8SAuditDataSourceRequestCount.WithLabelValues(ka.addr).Inc() } if r.Method != http.MethodPost { @@ -213,8 +200,8 @@ func (ka *KubernetesAuditSource) webhookHandler(w http.ResponseWriter, r *http.R remoteIP := strings.Split(r.RemoteAddr, ":")[0] for idx := range auditEvents.Items { - if ka.metricsLevel != configuration.METRICS_NONE { - eventCount.WithLabelValues(ka.addr).Inc() + if ka.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.K8SAuditDataSourceEventCount.With(prometheus.Labels{"source": ka.addr, "datasource_type": "k8s-audit", "acquis_type": ka.config.Labels["type"]}).Inc() } bytesEvent, err := json.Marshal(auditEvents.Items[idx]) diff --git a/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go b/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go index 868f75e10..7ff78a57b 100644 --- a/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go +++ b/pkg/acquisition/modules/kubernetesaudit/k8s_audit_test.go @@ -13,7 +13,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -91,7 +91,7 @@ webhook_path: /k8s-audit`, require.NoError(t, err) - err = f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err = f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) err = f.StreamingAcquisition(ctx, out, tb) @@ -259,7 +259,7 @@ webhook_path: /k8s-audit`, f := KubernetesAuditSource{} err := f.UnmarshalConfig([]byte(test.config)) require.NoError(t, err) - err = f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err = f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) diff --git a/pkg/acquisition/modules/loki/loki.go b/pkg/acquisition/modules/loki/loki.go index 28e2e92a6..ff3262865 100644 --- a/pkg/acquisition/modules/loki/loki.go +++ b/pkg/acquisition/modules/loki/loki.go @@ -20,6 +20,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki/internal/lokiclient" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -30,13 +31,6 @@ const ( lokiLimit int = 100 ) -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lokisource_hits_total", - Help: "Total lines that were read.", - }, - []string{"source"}) - type LokiAuthConfiguration struct { Username string `yaml:"username"` Password string `yaml:"password"` @@ -58,7 +52,7 @@ type LokiConfiguration struct { } type LokiSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config LokiConfiguration Client *lokiclient.LokiClient @@ -68,11 +62,11 @@ type LokiSource struct { } func (l *LokiSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.LokiDataSourceLinesRead} } func (l *LokiSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.LokiDataSourceLinesRead} } func (l *LokiSource) UnmarshalConfig(yamlConfig []byte) error { @@ -120,7 +114,7 @@ func (l *LokiSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (l *LokiSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { +func (l *LokiSource) Configure(config []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { l.Config = LokiConfiguration{} l.logger = logger l.metricsLevel = metricsLevel @@ -315,8 +309,8 @@ func (l *LokiSource) readOneEntry(entry lokiclient.Entry, labels map[string]stri ll.Process = true ll.Module = l.GetName() - if l.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": l.Config.URL}).Inc() + if l.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.LokiDataSourceLinesRead.With(prometheus.Labels{"source": l.Config.URL, "datasource_type": "loki", "acquis_type": ll.Labels["type"]}).Inc() } evt := types.MakeEvent(l.Config.UseTimeMachine, types.LOG, true) evt.Line = ll diff --git a/pkg/acquisition/modules/loki/loki_test.go b/pkg/acquisition/modules/loki/loki_test.go index 1f72f1394..f71aa5ba3 100644 --- a/pkg/acquisition/modules/loki/loki_test.go +++ b/pkg/acquisition/modules/loki/loki_test.go @@ -19,8 +19,8 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/loki" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -148,7 +148,7 @@ no_ready_check: 37 for _, test := range tests { t.Run(test.testName, func(t *testing.T) { lokiSource := loki.LokiSource{} - err := lokiSource.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := lokiSource.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.AssertErrorContains(t, err, test.expectedErr) if test.password != "" { @@ -372,7 +372,7 @@ since: 1h subLogger := logger.WithField("type", "loki") lokiSource := loki.LokiSource{} - if err := lokiSource.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE); err != nil { + if err := lokiSource.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone); err != nil { t.Fatalf("Unexpected error : %s", err) } @@ -459,7 +459,7 @@ query: > lokiTomb := tomb.Tomb{} lokiSource := loki.LokiSource{} - err := lokiSource.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := lokiSource.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } @@ -535,7 +535,7 @@ query: > title := time.Now().String() lokiSource := loki.LokiSource{} - err := lokiSource.Configure([]byte(config), subLogger, configuration.METRICS_NONE) + err := lokiSource.Configure([]byte(config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } diff --git a/pkg/acquisition/modules/s3/s3.go b/pkg/acquisition/modules/s3/s3.go index 0b8d14671..5d35d88d9 100644 --- a/pkg/acquisition/modules/s3/s3.go +++ b/pkg/acquisition/modules/s3/s3.go @@ -28,6 +28,7 @@ import ( "gopkg.in/tomb.v2" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -47,7 +48,7 @@ type S3Configuration struct { } type S3Source struct { - MetricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config S3Configuration logger *log.Entry s3Client s3iface.S3API @@ -107,30 +108,6 @@ const ( SQSFormatSNS = "sns" ) -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_s3_hits_total", - Help: "Number of events read per bucket.", - }, - []string{"bucket"}, -) - -var objectsRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_s3_objects_total", - Help: "Number of objects read per bucket.", - }, - []string{"bucket"}, -) - -var sqsMessagesReceived = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_s3_sqs_messages_total", - Help: "Number of SQS messages received per queue.", - }, - []string{"queue"}, -) - func (s *S3Source) newS3Client() error { options := session.Options{ SharedConfigState: session.SharedConfigEnable, @@ -375,8 +352,8 @@ func (s *S3Source) sqsPoll() error { logger.Tracef("SQS output: %v", out) logger.Debugf("Received %d messages from SQS", len(out.Messages)) for _, message := range out.Messages { - if s.MetricsLevel != configuration.METRICS_NONE { - sqsMessagesReceived.WithLabelValues(s.Config.SQSName).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.S3DataSourceSQSMessagesReceived.WithLabelValues(s.Config.SQSName).Inc() } bucket, key, err := s.extractBucketAndPrefix(message.Body) if err != nil { @@ -457,8 +434,8 @@ func (s *S3Source) readFile(bucket string, key string) error { default: text := scanner.Text() logger.Tracef("Read line %s", text) - if s.MetricsLevel != configuration.METRICS_NONE { - linesRead.WithLabelValues(bucket).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.S3DataSourceLinesRead.With(prometheus.Labels{"bucket": bucket, "datasource_type": "s3", "acquis_type": s.Config.Labels["type"]}).Inc() } l := types.Line{} l.Raw = text @@ -466,9 +443,10 @@ func (s *S3Source) readFile(bucket string, key string) error { l.Time = time.Now().UTC() l.Process = true l.Module = s.GetName() - if s.MetricsLevel == configuration.METRICS_FULL { + switch s.metricsLevel { + case metrics.AcquisitionMetricsLevelFull: l.Src = bucket + "/" + key - } else if s.MetricsLevel == configuration.METRICS_AGGREGATE { + case metrics.AcquisitionMetricsLevelAggregated, metrics.AcquisitionMetricsLevelNone: // Even if metrics are disabled, we want to source in the event l.Src = bucket } evt := types.MakeEvent(s.Config.UseTimeMachine, types.LOG, true) @@ -479,8 +457,8 @@ func (s *S3Source) readFile(bucket string, key string) error { if err := scanner.Err(); err != nil { return fmt.Errorf("failed to read object %s/%s: %s", bucket, key, err) } - if s.MetricsLevel != configuration.METRICS_NONE { - objectsRead.WithLabelValues(bucket).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.S3DataSourceObjectsRead.WithLabelValues(bucket).Inc() } return nil } @@ -490,11 +468,11 @@ func (s *S3Source) GetUuid() string { } func (s *S3Source) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, objectsRead, sqsMessagesReceived} + return []prometheus.Collector{metrics.S3DataSourceLinesRead, metrics.S3DataSourceObjectsRead, metrics.S3DataSourceSQSMessagesReceived} } func (s *S3Source) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead, objectsRead, sqsMessagesReceived} + return []prometheus.Collector{metrics.S3DataSourceLinesRead, metrics.S3DataSourceObjectsRead, metrics.S3DataSourceSQSMessagesReceived} } func (s *S3Source) UnmarshalConfig(yamlConfig []byte) error { @@ -541,7 +519,7 @@ func (s *S3Source) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (s *S3Source) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (s *S3Source) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { err := s.UnmarshalConfig(yamlConfig) if err != nil { return err diff --git a/pkg/acquisition/modules/s3/s3_test.go b/pkg/acquisition/modules/s3/s3_test.go index 93f30f7d1..d62d0db1a 100644 --- a/pkg/acquisition/modules/s3/s3_test.go +++ b/pkg/acquisition/modules/s3/s3_test.go @@ -20,7 +20,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -79,7 +79,7 @@ sqs_name: foobar t.Run(test.name, func(t *testing.T) { f := S3Source{} - err := f.Configure([]byte(test.config), nil, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), nil, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, test.expectedErr) }) } @@ -120,7 +120,7 @@ polling_method: list f := S3Source{} logger := log.NewEntry(log.New()) - err := f.Configure([]byte(test.config), logger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), logger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } @@ -340,7 +340,7 @@ prefix: foo/ f := S3Source{} logger := log.NewEntry(log.New()) logger.Logger.SetLevel(log.TraceLevel) - err := f.Configure([]byte(test.config), logger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), logger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } @@ -425,7 +425,7 @@ sqs_name: test linesRead := 0 f := S3Source{} logger := log.NewEntry(log.New()) - err := f.Configure([]byte(test.config), logger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), logger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } diff --git a/pkg/acquisition/modules/syslog/syslog.go b/pkg/acquisition/modules/syslog/syslog.go index b679544ce..826197448 100644 --- a/pkg/acquisition/modules/syslog/syslog.go +++ b/pkg/acquisition/modules/syslog/syslog.go @@ -20,6 +20,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc3164" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/parser/rfc5424" syslogserver "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/syslog/internal/server" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -33,27 +34,13 @@ type SyslogConfiguration struct { } type SyslogSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config SyslogConfiguration logger *log.Entry server *syslogserver.SyslogServer serverTomb *tomb.Tomb } -var linesReceived = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_syslogsource_hits_total", - Help: "Total lines that were received.", - }, - []string{"source"}) - -var linesParsed = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_syslogsource_parsed_total", - Help: "Total lines that were successfully parsed", - }, - []string{"source", "type"}) - func (s *SyslogSource) GetUuid() string { return s.config.UniqueId } @@ -75,11 +62,11 @@ func (s *SyslogSource) CanRun() error { } func (s *SyslogSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesReceived, linesParsed} + return []prometheus.Collector{metrics.SyslogDataSourceLinesReceived, metrics.SyslogDataSourceLinesParsed} } func (s *SyslogSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesReceived, linesParsed} + return []prometheus.Collector{metrics.SyslogDataSourceLinesReceived, metrics.SyslogDataSourceLinesParsed} } func (s *SyslogSource) ConfigureByDSN(dsn string, labels map[string]string, logger *log.Entry, uuid string) error { @@ -126,7 +113,7 @@ func (s *SyslogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (s *SyslogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { s.logger = logger s.logger.Infof("Starting syslog datasource configuration") s.metricsLevel = metricsLevel @@ -189,8 +176,8 @@ func (s *SyslogSource) parseLine(syslogLine syslogserver.SyslogMessage) string { logger := s.logger.WithField("client", syslogLine.Client) logger.Tracef("raw: %s", syslogLine) - if s.metricsLevel != configuration.METRICS_NONE { - linesReceived.With(prometheus.Labels{"source": syslogLine.Client}).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.SyslogDataSourceLinesReceived.With(prometheus.Labels{"source": syslogLine.Client, "datasource_type": "syslog", "acquis_type": s.config.Labels["type"]}).Inc() } if !s.config.DisableRFCParser { p := rfc3164.NewRFC3164Parser(rfc3164.WithCurrentYear()) @@ -205,13 +192,13 @@ func (s *SyslogSource) parseLine(syslogLine syslogserver.SyslogMessage) string { return "" } line = s.buildLogFromSyslog(p2.Timestamp, p2.Hostname, p2.Tag, p2.PID, p2.Message) - if s.metricsLevel != configuration.METRICS_NONE { - linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc5424"}).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.SyslogDataSourceLinesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc5424", "datasource_type": "syslog", "acquis_type": s.config.Labels["type"]}).Inc() } } else { line = s.buildLogFromSyslog(p.Timestamp, p.Hostname, p.Tag, p.PID, p.Message) - if s.metricsLevel != configuration.METRICS_NONE { - linesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc3164"}).Inc() + if s.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.SyslogDataSourceLinesParsed.With(prometheus.Labels{"source": syslogLine.Client, "type": "rfc3164", "datasource_type": "syslog", "acquis_type": s.config.Labels["type"]}).Inc() } } } else { diff --git a/pkg/acquisition/modules/syslog/syslog_test.go b/pkg/acquisition/modules/syslog/syslog_test.go index b6f78cdbe..2f5785597 100644 --- a/pkg/acquisition/modules/syslog/syslog_test.go +++ b/pkg/acquisition/modules/syslog/syslog_test.go @@ -14,7 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -57,7 +57,7 @@ listen_addr: 10.0.0`, for _, test := range tests { t.Run(test.config, func(t *testing.T) { s := SyslogSource{} - err := s.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := s.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.AssertErrorContains(t, err, test.expectedErr) }) } @@ -161,7 +161,7 @@ disable_rfc_parser: true`, t.Run(ts.name, func(t *testing.T) { subLogger := log.WithField("type", "syslog") s := SyslogSource{} - err := s.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := s.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("could not configure syslog source : %s", err) } diff --git a/pkg/acquisition/modules/victorialogs/victorialogs.go b/pkg/acquisition/modules/victorialogs/victorialogs.go index 801748730..5fdfc62f8 100644 --- a/pkg/acquisition/modules/victorialogs/victorialogs.go +++ b/pkg/acquisition/modules/victorialogs/victorialogs.go @@ -16,6 +16,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/victorialogs/internal/vlclient" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -23,13 +24,6 @@ const ( defaultLimit int = 100 ) -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_victorialogssource_hits_total", - Help: "Total lines that were read.", - }, - []string{"source"}) - type VLAuthConfiguration struct { Username string `yaml:"username"` Password string `yaml:"password"` @@ -49,7 +43,7 @@ type VLConfiguration struct { } type VLSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel Config VLConfiguration Client *vlclient.VLClient @@ -58,11 +52,11 @@ type VLSource struct { } func (l *VLSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.VictorialogsDataSourceLinesRead} } func (l *VLSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.VictorialogsDataSourceLinesRead} } func (l *VLSource) UnmarshalConfig(yamlConfig []byte) error { @@ -110,7 +104,7 @@ func (l *VLSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (l *VLSource) Configure(config []byte, logger *log.Entry, metricsLevel int) error { +func (l *VLSource) Configure(config []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { l.Config = VLConfiguration{} l.logger = logger l.metricsLevel = metricsLevel @@ -279,8 +273,8 @@ func (l *VLSource) readOneEntry(entry *vlclient.Log, labels map[string]string, o ll.Process = true ll.Module = l.GetName() - if l.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": l.Config.URL}).Inc() + if l.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.VictorialogsDataSourceLinesRead.With(prometheus.Labels{"source": l.Config.URL, "datasource_type": "victorialogs", "acquis_type": l.Config.Labels["type"]}).Inc() } expectMode := types.LIVE if l.Config.UseTimeMachine { diff --git a/pkg/acquisition/modules/victorialogs/victorialogs_test.go b/pkg/acquisition/modules/victorialogs/victorialogs_test.go index 93948799c..6dc67536e 100644 --- a/pkg/acquisition/modules/victorialogs/victorialogs_test.go +++ b/pkg/acquisition/modules/victorialogs/victorialogs_test.go @@ -20,8 +20,8 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" "github.com/crowdsecurity/crowdsec/pkg/acquisition/modules/victorialogs" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -113,7 +113,7 @@ query: > for _, test := range tests { t.Run(test.testName, func(t *testing.T) { vlSource := victorialogs.VLSource{} - err := vlSource.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := vlSource.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.AssertErrorContains(t, err, test.expectedErr) if test.password != "" { @@ -294,7 +294,7 @@ since: 1h subLogger := logger.WithField("type", "victorialogs") vlSource := victorialogs.VLSource{} - err := vlSource.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := vlSource.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } @@ -380,7 +380,7 @@ query: > vlTomb := tomb.Tomb{} vlSource := victorialogs.VLSource{} - err := vlSource.Configure([]byte(ts.config), subLogger, configuration.METRICS_NONE) + err := vlSource.Configure([]byte(ts.config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } @@ -454,7 +454,7 @@ query: > title := time.Now().String() vlSource := victorialogs.VLSource{} - err := vlSource.Configure([]byte(config), subLogger, configuration.METRICS_NONE) + err := vlSource.Configure([]byte(config), subLogger, metrics.AcquisitionMetricsLevelNone) if err != nil { t.Fatalf("Unexpected error : %s", err) } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog.go b/pkg/acquisition/modules/wineventlog/wineventlog.go index 3023a3715..a277fa2c8 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog.go @@ -11,6 +11,7 @@ import ( "gopkg.in/tomb.v2" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -24,7 +25,7 @@ func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { return nil } diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go index 2d45cf249..3cd2ae3b0 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows.go @@ -23,6 +23,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -37,7 +38,7 @@ type WinEventLogConfiguration struct { } type WinEventLogSource struct { - metricsLevel int + metricsLevel metrics.AcquisitionMetricsLevel config WinEventLogConfiguration logger *log.Entry evtConfig *winlog.SubscribeConfig @@ -57,13 +58,6 @@ type Select struct { // 0 identifies the local machine in windows APIs const localMachine = 0 -var linesRead = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_winevtlogsource_hits_total", - Help: "Total event that were read.", - }, - []string{"source"}) - func logLevelToInt(logLevel string) ([]string, error) { switch strings.ToUpper(logLevel) { case "CRITICAL": @@ -195,8 +189,8 @@ func (w *WinEventLogSource) getEvents(out chan types.Event, t *tomb.Tomb) error continue } for _, event := range renderedEvents { - if w.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": w.name}).Inc() + if w.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.WineventlogDataSourceLinesRead.With(prometheus.Labels{"source": w.name, "datasource_type": "wineventlog", "acquis_type": w.config.Labels["type"]}).Inc() } l := types.Line{} l.Raw = event @@ -287,7 +281,7 @@ func (w *WinEventLogSource) UnmarshalConfig(yamlConfig []byte) error { return nil } -func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel int) error { +func (w *WinEventLogSource) Configure(yamlConfig []byte, logger *log.Entry, metricsLevel metrics.AcquisitionMetricsLevel) error { w.logger = logger w.metricsLevel = metricsLevel @@ -420,8 +414,8 @@ OUTER_LOOP: w.logger.Debugf("Got %d events", len(evts)) for _, evt := range evts { w.logger.Tracef("Event: %s", evt) - if w.metricsLevel != configuration.METRICS_NONE { - linesRead.With(prometheus.Labels{"source": w.name}).Inc() + if w.metricsLevel != metrics.AcquisitionMetricsLevelNone { + metrics.WineventlogDataSourceLinesRead.With(prometheus.Labels{"source": w.name, "datasource_type": "wineventlog", "acquis_type": w.config.Labels["type"]}).Inc() } l := types.Line{} l.Raw = evt @@ -441,11 +435,11 @@ OUTER_LOOP: } func (w *WinEventLogSource) GetMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.WineventlogDataSourceLinesRead} } func (w *WinEventLogSource) GetAggregMetrics() []prometheus.Collector { - return []prometheus.Collector{linesRead} + return []prometheus.Collector{metrics.WineventlogDataSourceLinesRead} } func (w *WinEventLogSource) GetName() string { diff --git a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go index e8a096771..6c9f65fd8 100644 --- a/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go +++ b/pkg/acquisition/modules/wineventlog/wineventlog_windows_test.go @@ -14,7 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/cstest" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -64,7 +64,7 @@ event_ids: true`, subLogger := log.WithField("type", "windowseventlog") for _, test := range tests { f := WinEventLogSource{} - err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) assert.Contains(t, err.Error(), test.expectedErr) } } @@ -123,7 +123,7 @@ event_level: bla`, t.Run(test.config, func(t *testing.T) { f := WinEventLogSource{} - err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) cstest.RequireErrorContains(t, err, test.expectedErr) if test.expectedErr != "" { return @@ -198,7 +198,7 @@ event_ids: c := make(chan types.Event) f := WinEventLogSource{} - err := f.Configure([]byte(test.config), subLogger, configuration.METRICS_NONE) + err := f.Configure([]byte(test.config), subLogger, metrics.AcquisitionMetricsLevelNone) require.NoError(t, err) err = f.StreamingAcquisition(ctx, c, to) diff --git a/pkg/apiclient/client_http.go b/pkg/apiclient/client_http.go index d3aaa6bc4..a1ffc52a2 100644 --- a/pkg/apiclient/client_http.go +++ b/pkg/apiclient/client_http.go @@ -2,6 +2,7 @@ package apiclient import ( "bytes" + "compress/gzip" "context" "encoding/json" "errors" @@ -15,6 +16,8 @@ import ( log "github.com/sirupsen/logrus" ) +const compressionMinSize = 5 * 1024 // 5KB + func (c *ApiClient) PrepareRequest(ctx context.Context, method, url string, body any) (*http.Request, error) { if !strings.HasSuffix(c.BaseURL.Path, "/") { return nil, fmt.Errorf("BaseURL must have a trailing slash, but %q does not", c.BaseURL) @@ -26,14 +29,32 @@ func (c *ApiClient) PrepareRequest(ctx context.Context, method, url string, body } var buf io.ReadWriter + compressedBody := false + if body != nil { - buf = &bytes.Buffer{} - enc := json.NewEncoder(buf) + jsonBuf := &bytes.Buffer{} + enc := json.NewEncoder(jsonBuf) enc.SetEscapeHTML(false) if err = enc.Encode(body); err != nil { return nil, err } + + jsonBytes := jsonBuf.Bytes() + if len(jsonBytes) > compressionMinSize { + compressedBody = true + buf = &bytes.Buffer{} + gzipWriter := gzip.NewWriter(buf) + if _, err = gzipWriter.Write(jsonBytes); err != nil { + return nil, fmt.Errorf("writing to gzip writer: %w", err) + } + if err = gzipWriter.Close(); err != nil { + return nil, fmt.Errorf("closing gzip writer: %w", err) + } + } else { + buf = jsonBuf + } + } req, err := http.NewRequestWithContext(ctx, method, u.String(), buf) @@ -43,6 +64,9 @@ func (c *ApiClient) PrepareRequest(ctx context.Context, method, url string, body if body != nil { req.Header.Set("Content-Type", "application/json") + if compressedBody { + req.Header.Set("Content-Encoding", "gzip") + } } return req, nil diff --git a/pkg/apiserver/apic_metrics.go b/pkg/apiserver/apic_metrics.go index df3148925..ede66f6b9 100644 --- a/pkg/apiserver/apic_metrics.go +++ b/pkg/apiserver/apic_metrics.go @@ -81,7 +81,7 @@ func (a *apic) GetUsageMetrics(ctx context.Context) (*models.AllMetrics, []int, } for _, lp := range lps { - dbMetrics, err := a.dbClient.GetLPUsageMetricsByMachineID(ctx, lp.MachineId) + dbMetrics, err := a.dbClient.GetLPUsageMetricsByMachineID(ctx, lp.MachineId, true) if err != nil { log.Errorf("unable to get LP usage metrics: %s", err) continue diff --git a/pkg/apiserver/controllers/controller.go b/pkg/apiserver/controllers/controller.go index 9166b0fc8..3fb472956 100644 --- a/pkg/apiserver/controllers/controller.go +++ b/pkg/apiserver/controllers/controller.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/alexliesenfeld/health" + "github.com/gin-contrib/gzip" "github.com/gin-gonic/gin" log "github.com/sirupsen/logrus" @@ -96,6 +97,9 @@ func (c *Controller) NewV1() error { c.Router.GET("/health", gin.WrapF(serveHealth())) c.Router.Use(v1.PrometheusMiddleware()) + // We don't want to compress the response body as it would likely break some existing bouncers + // But we do want to automatically uncompress incoming requests + c.Router.Use(gzip.Gzip(gzip.NoCompression, gzip.WithDecompressOnly(), gzip.WithDecompressFn(gzip.DefaultDecompressHandle))) c.Router.HandleMethodNotAllowed = true c.Router.UnescapePathValues = true c.Router.UseRawPath = true diff --git a/pkg/apiserver/controllers/v1/metrics.go b/pkg/apiserver/controllers/v1/metrics.go index 4f6ee0986..bf16e3dd6 100644 --- a/pkg/apiserver/controllers/v1/metrics.go +++ b/pkg/apiserver/controllers/v1/metrics.go @@ -3,71 +3,15 @@ package v1 import ( "time" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/gin-gonic/gin" "github.com/prometheus/client_golang/prometheus" ) -/*prometheus*/ -var LapiRouteHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lapi_route_requests_total", - Help: "Number of calls to each route per method.", - }, - []string{"route", "method"}, -) - -/*hits per machine*/ -var LapiMachineHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lapi_machine_requests_total", - Help: "Number of calls to each route per method grouped by machines.", - }, - []string{"machine", "route", "method"}, -) - -/*hits per bouncer*/ -var LapiBouncerHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lapi_bouncer_requests_total", - Help: "Number of calls to each route per method grouped by bouncers.", - }, - []string{"bouncer", "route", "method"}, -) - -/* - keep track of the number of calls (per bouncer) that lead to nil/non-nil responses. - -while it's not exact, it's a good way to know - when you have a rutpure bouncer - what is the rate of ok/ko answers you got from lapi -*/ -var LapiNilDecisions = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lapi_decisions_ko_total", - Help: "Number of calls to /decisions that returned nil result.", - }, - []string{"bouncer"}, -) - -/*hits per bouncer*/ -var LapiNonNilDecisions = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_lapi_decisions_ok_total", - Help: "Number of calls to /decisions that returned non-nil result.", - }, - []string{"bouncer"}, -) - -var LapiResponseTime = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "cs_lapi_request_duration_seconds", - Help: "Response time of LAPI", - Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1}, - }, - []string{"endpoint", "method"}) - func PrometheusBouncersHasEmptyDecision(c *gin.Context) { bouncer, _ := getBouncerFromContext(c) if bouncer != nil { - LapiNilDecisions.With(prometheus.Labels{ + metrics.LapiNilDecisions.With(prometheus.Labels{ "bouncer": bouncer.Name, }).Inc() } @@ -76,7 +20,7 @@ func PrometheusBouncersHasEmptyDecision(c *gin.Context) { func PrometheusBouncersHasNonEmptyDecision(c *gin.Context) { bouncer, _ := getBouncerFromContext(c) if bouncer != nil { - LapiNonNilDecisions.With(prometheus.Labels{ + metrics.LapiNonNilDecisions.With(prometheus.Labels{ "bouncer": bouncer.Name, }).Inc() } @@ -86,7 +30,7 @@ func PrometheusMachinesMiddleware() gin.HandlerFunc { return func(c *gin.Context) { machineID, _ := getMachineIDFromContext(c) if machineID != "" { - LapiMachineHits.With(prometheus.Labels{ + metrics.LapiMachineHits.With(prometheus.Labels{ "machine": machineID, "route": c.Request.URL.Path, "method": c.Request.Method, @@ -101,7 +45,7 @@ func PrometheusBouncersMiddleware() gin.HandlerFunc { return func(c *gin.Context) { bouncer, _ := getBouncerFromContext(c) if bouncer != nil { - LapiBouncerHits.With(prometheus.Labels{ + metrics.LapiBouncerHits.With(prometheus.Labels{ "bouncer": bouncer.Name, "route": c.Request.URL.Path, "method": c.Request.Method, @@ -116,13 +60,13 @@ func PrometheusMiddleware() gin.HandlerFunc { return func(c *gin.Context) { startTime := time.Now() - LapiRouteHits.With(prometheus.Labels{ + metrics.LapiRouteHits.With(prometheus.Labels{ "route": c.Request.URL.Path, "method": c.Request.Method, }).Inc() c.Next() elapsed := time.Since(startTime) - LapiResponseTime.With(prometheus.Labels{"method": c.Request.Method, "endpoint": c.Request.URL.Path}).Observe(elapsed.Seconds()) + metrics.LapiResponseTime.With(prometheus.Labels{"method": c.Request.Method, "endpoint": c.Request.URL.Path}).Observe(elapsed.Seconds()) } } diff --git a/pkg/apiserver/usage_metrics_test.go b/pkg/apiserver/usage_metrics_test.go index 82b5fc171..4647d9cea 100644 --- a/pkg/apiserver/usage_metrics_test.go +++ b/pkg/apiserver/usage_metrics_test.go @@ -199,7 +199,7 @@ func TestLPMetrics(t *testing.T) { assert.Contains(t, w.Body.String(), tt.expectedResponse) machine, _ := dbClient.QueryMachineByID(ctx, "test") - metrics, _ := dbClient.GetLPUsageMetricsByMachineID(ctx, "test") + metrics, _ := dbClient.GetLPUsageMetricsByMachineID(ctx, "test", true) assert.Len(t, metrics, tt.expectedMetricsCount) assert.Equal(t, tt.expectedOSName, machine.Osname) diff --git a/pkg/cache/cache.go b/pkg/cache/cache.go index 3885294c5..ac06a9b08 100644 --- a/pkg/cache/cache.go +++ b/pkg/cache/cache.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -18,21 +19,12 @@ var ( CacheConfig []CacheCfg ) -/*prometheus*/ -var CacheMetrics = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_cache_size", - Help: "Entries per cache.", - }, - []string{"name", "type"}, -) - // UpdateCacheMetrics is called directly by the prom handler func UpdateCacheMetrics() { - CacheMetrics.Reset() + metrics.CacheMetrics.Reset() for i, name := range CacheNames { - CacheMetrics.With(prometheus.Labels{"name": name, "type": CacheConfig[i].Strategy}).Set(float64(Caches[i].Len(false))) + metrics.CacheMetrics.With(prometheus.Labels{"name": name, "type": CacheConfig[i].Strategy}).Set(float64(Caches[i].Len(false))) } } diff --git a/pkg/csconfig/config.go b/pkg/csconfig/config.go index fc69146ea..077d2e4ff 100644 --- a/pkg/csconfig/config.go +++ b/pkg/csconfig/config.go @@ -16,7 +16,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/csyaml" "github.com/crowdsecurity/go-cs-lib/ptr" - "github.com/crowdsecurity/crowdsec/pkg/acquisition/configuration" + "github.com/crowdsecurity/crowdsec/pkg/metrics" ) // defaultConfigDir is the base path to all configuration files, to be overridden in the Makefile */ @@ -120,7 +120,7 @@ func NewDefaultConfig() *Config { } prometheus := PrometheusCfg{ Enabled: true, - Level: configuration.CFG_METRICS_FULL, + Level: metrics.MetricsLevelFull, } configPaths := ConfigurationPaths{ ConfigDir: DefaultConfigPath("."), diff --git a/pkg/csconfig/prometheus.go b/pkg/csconfig/prometheus.go index 9b80fe398..ca705e124 100644 --- a/pkg/csconfig/prometheus.go +++ b/pkg/csconfig/prometheus.go @@ -1,8 +1,10 @@ package csconfig +import "github.com/crowdsecurity/crowdsec/pkg/metrics" + type PrometheusCfg struct { - Enabled bool `yaml:"enabled"` - Level string `yaml:"level"` //aggregated|full - ListenAddr string `yaml:"listen_addr"` - ListenPort int `yaml:"listen_port"` + Enabled bool `yaml:"enabled"` + Level metrics.MetricsLevelConfig `yaml:"level"` + ListenAddr string `yaml:"listen_addr"` + ListenPort int `yaml:"listen_port"` } diff --git a/pkg/database/metrics.go b/pkg/database/metrics.go index eb4c47282..f2400b8d5 100644 --- a/pkg/database/metrics.go +++ b/pkg/database/metrics.go @@ -25,14 +25,18 @@ func (c *Client) CreateMetric(ctx context.Context, generatedType metric.Generate return metric, nil } -func (c *Client) GetLPUsageMetricsByMachineID(ctx context.Context, machineId string) ([]*ent.Metric, error) { - metrics, err := c.Ent.Metric.Query(). +func (c *Client) GetLPUsageMetricsByMachineID(ctx context.Context, machineId string, toSend bool) ([]*ent.Metric, error) { + query := c.Ent.Metric.Query(). Where( metric.GeneratedTypeEQ(metric.GeneratedTypeLP), metric.GeneratedByEQ(machineId), - metric.PushedAtIsNil(), - ). - All(ctx) + ) + + if toSend { + query = query.Where(metric.PushedAtIsNil()) + } + + metrics, err := query.All(ctx) if err != nil { c.Log.Warningf("GetLPUsageMetricsByOrigin: %s", err) return nil, fmt.Errorf("getting LP usage metrics by origin %s: %w", machineId, err) diff --git a/pkg/exprhelpers/helpers.go b/pkg/exprhelpers/helpers.go index 613334b7a..1c467787b 100644 --- a/pkg/exprhelpers/helpers.go +++ b/pkg/exprhelpers/helpers.go @@ -35,6 +35,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/database" "github.com/crowdsecurity/crowdsec/pkg/fflag" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -47,15 +48,6 @@ var ( // This is used to (optionally) cache regexp results for RegexpInFile operations var dataFileRegexCache map[string]gcache.Cache = make(map[string]gcache.Cache) -/*prometheus*/ -var RegexpCacheMetrics = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_regexp_cache_size", - Help: "Entries per regexp cache.", - }, - []string{"name"}, -) - var dbClient *database.Client var exprFunctionOptions []expr.Option @@ -188,10 +180,10 @@ func RegexpCacheInit(filename string, cacheCfg types.DataSource) error { // UpdateCacheMetrics is called directly by the prom handler func UpdateRegexpCacheMetrics() { - RegexpCacheMetrics.Reset() + metrics.RegexpCacheMetrics.Reset() for name := range dataFileRegexCache { - RegexpCacheMetrics.With(prometheus.Labels{"name": name}).Set(float64(dataFileRegexCache[name].Len(true))) + metrics.RegexpCacheMetrics.With(prometheus.Labels{"name": name}).Set(float64(dataFileRegexCache[name].Len(true))) } } diff --git a/pkg/leakybucket/bucket.go b/pkg/leakybucket/bucket.go index e7ea6e3e2..d14e42adf 100644 --- a/pkg/leakybucket/bucket.go +++ b/pkg/leakybucket/bucket.go @@ -14,6 +14,7 @@ import ( "github.com/crowdsecurity/go-cs-lib/trace" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/time/rate" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -74,54 +75,6 @@ type Leaky struct { orderEvent bool } -var BucketsPour = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_bucket_poured_total", - Help: "Total events were poured in bucket.", - }, - []string{"source", "type", "name"}, -) - -var BucketsOverflow = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_bucket_overflowed_total", - Help: "Total buckets overflowed.", - }, - []string{"name"}, -) - -var BucketsCanceled = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_bucket_canceled_total", - Help: "Total buckets canceled.", - }, - []string{"name"}, -) - -var BucketsUnderflow = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_bucket_underflowed_total", - Help: "Total buckets underflowed.", - }, - []string{"name"}, -) - -var BucketsInstantiation = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_bucket_created_total", - Help: "Total buckets were instantiated.", - }, - []string{"name"}, -) - -var BucketsCurrentCount = prometheus.NewGaugeVec( - prometheus.GaugeOpts{ - Name: "cs_buckets", - Help: "Number of buckets that currently exist.", - }, - []string{"name"}, -) - var LeakyRoutineCount int64 // Newleaky creates a new leaky bucket from a BucketFactory @@ -153,7 +106,7 @@ func FromFactory(bucketFactory BucketFactory) *Leaky { } else { limiter = rate.NewLimiter(rate.Every(bucketFactory.leakspeed), bucketFactory.Capacity) } - BucketsInstantiation.With(prometheus.Labels{"name": bucketFactory.Name}).Inc() + metrics.BucketsInstantiation.With(prometheus.Labels{"name": bucketFactory.Name}).Inc() //create the leaky bucket per se l := &Leaky{ @@ -212,8 +165,8 @@ func LeakRoutine(leaky *Leaky) error { defer trace.CatchPanic(fmt.Sprintf("crowdsec/LeakRoutine/%s", leaky.Name)) - BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Inc() - defer BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Dec() + metrics.BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Inc() + defer metrics.BucketsCurrentCount.With(prometheus.Labels{"name": leaky.Name}).Dec() /*todo : we create a logger at runtime while we want leakroutine to be up asap, might not be a good idea*/ leaky.logger = leaky.BucketConfig.logger.WithFields(log.Fields{"partition": leaky.Mapkey, "bucket_id": leaky.Uuid}) @@ -256,7 +209,7 @@ func LeakRoutine(leaky *Leaky) error { if leaky.logger.Level >= log.TraceLevel { leaky.logger.Tracef("Pour event: %s", spew.Sdump(msg)) } - BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc() + metrics.BucketsPour.With(prometheus.Labels{"name": leaky.Name, "source": msg.Line.Src, "type": msg.Line.Module}).Inc() leaky.Pour(leaky, *msg) // glue for now @@ -295,7 +248,7 @@ func LeakRoutine(leaky *Leaky) error { /*suiciiiide*/ case <-leaky.Suicide: close(leaky.Signal) - BucketsCanceled.With(prometheus.Labels{"name": leaky.Name}).Inc() + metrics.BucketsCanceled.With(prometheus.Labels{"name": leaky.Name}).Inc() leaky.logger.Debugf("Suicide triggered") leaky.AllOut <- types.Event{Type: types.OVFLW, Overflow: types.RuntimeAlert{Mapkey: leaky.Mapkey}} leaky.logger.Tracef("Returning from leaky routine.") @@ -312,7 +265,7 @@ func LeakRoutine(leaky *Leaky) error { alert = types.RuntimeAlert{Mapkey: leaky.Mapkey} if leaky.timedOverflow { - BucketsOverflow.With(prometheus.Labels{"name": leaky.Name}).Inc() + metrics.BucketsOverflow.With(prometheus.Labels{"name": leaky.Name}).Inc() alert, err = NewAlert(leaky, ofw) if err != nil { @@ -328,7 +281,7 @@ func LeakRoutine(leaky *Leaky) error { leaky.logger.Infof("Timed Overflow") } else { leaky.logger.Debugf("bucket underflow, destroy") - BucketsUnderflow.With(prometheus.Labels{"name": leaky.Name}).Inc() + metrics.BucketsUnderflow.With(prometheus.Labels{"name": leaky.Name}).Inc() } if leaky.logger.Level >= log.TraceLevel { @@ -394,7 +347,7 @@ func (leaky *Leaky) overflow(ofw *types.Queue) { mt, _ := leaky.Ovflw_ts.MarshalText() leaky.logger.Tracef("overflow time : %s", mt) - BucketsOverflow.With(prometheus.Labels{"name": leaky.Name}).Inc() + metrics.BucketsOverflow.With(prometheus.Labels{"name": leaky.Name}).Inc() leaky.AllOut <- types.Event{Overflow: alert, Type: types.OVFLW, MarshaledTime: string(mt)} } diff --git a/pkg/leakybucket/manager_run.go b/pkg/leakybucket/manager_run.go index e6712e6e4..49c6ef45e 100644 --- a/pkg/leakybucket/manager_run.go +++ b/pkg/leakybucket/manager_run.go @@ -14,6 +14,7 @@ import ( log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -56,7 +57,7 @@ func GarbageCollectBuckets(deadline time.Time, buckets *Buckets) error { tokcapa = math.Round(tokcapa*100) / 100 //bucket actually underflowed based on log time, but no in real time if tokat >= tokcapa { - BucketsUnderflow.With(prometheus.Labels{"name": val.Name}).Inc() + metrics.BucketsUnderflow.With(prometheus.Labels{"name": val.Name}).Inc() val.logger.Debugf("UNDERFLOW : first_ts:%s tokens_at:%f capcity:%f", val.First_ts, tokat, tokcapa) toflush = append(toflush, key) val.tomb.Kill(nil) @@ -116,7 +117,7 @@ func DumpBucketsStateAt(deadline time.Time, outputdir string, buckets *Buckets) tokcapa = math.Round(tokcapa*100) / 100 if tokat >= tokcapa { - BucketsUnderflow.With(prometheus.Labels{"name": val.Name}).Inc() + metrics.BucketsUnderflow.With(prometheus.Labels{"name": val.Name}).Inc() val.logger.Debugf("UNDERFLOW : first_ts:%s tokens_at:%f capcity:%f", val.First_ts, tokat, tokcapa) discard += 1 return true diff --git a/pkg/metrics/acquisition.go b/pkg/metrics/acquisition.go new file mode 100644 index 000000000..2ebbbd461 --- /dev/null +++ b/pkg/metrics/acquisition.go @@ -0,0 +1,7 @@ +package metrics + +var AcquisitionMetricsNames = []string{} + +func RegisterAcquisitionMetric(metricName string) { + AcquisitionMetricsNames = append(AcquisitionMetricsNames, metricName) +} diff --git a/pkg/metrics/acquisition_appsec.go b/pkg/metrics/acquisition_appsec.go new file mode 100644 index 000000000..45f34b7da --- /dev/null +++ b/pkg/metrics/acquisition_appsec.go @@ -0,0 +1,68 @@ +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const AppsecGlobalParsingHistogramMetricName = "cs_appsec_parsing_time_seconds" + +var AppsecGlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: AppsecGlobalParsingHistogramMetricName, + Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, + }, + []string{"source", "appsec_engine"}, +) + +const AppsecInbandParsingHistogramMetricName = "cs_appsec_inband_parsing_time_seconds" + +var AppsecInbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the inband Application Security Engine.", + Name: AppsecInbandParsingHistogramMetricName, + Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, + }, + []string{"source", "appsec_engine"}, +) + +const AppsecOutbandParsingHistogramMetricName = "cs_appsec_outband_parsing_time_seconds" + +var AppsecOutbandParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent processing a request by the Application Security Engine.", + Name: AppsecOutbandParsingHistogramMetricName, + Buckets: []float64{0.0001, 0.00025, 0.0005, 0.001, 0.0025, 0.0050, 0.01, 0.025, 0.05, 0.1, 0.25}, + }, + []string{"source", "appsec_engine"}, +) + +const AppsecReqCounterMetricName = "cs_appsec_reqs_total" + +var AppsecReqCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: AppsecReqCounterMetricName, + Help: "Total events processed by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +const AppsecBlockCounterMetricName = "cs_appsec_block_total" + +var AppsecBlockCounter = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: AppsecBlockCounterMetricName, + Help: "Total events blocked by the Application Security Engine.", + }, + []string{"source", "appsec_engine"}, +) + +const AppsecRuleHitsMetricName = "cs_appsec_rule_hits" + +var AppsecRuleHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: AppsecRuleHitsMetricName, + Help: "Count of triggered rule, by rule_name, type (inband/outofband), appsec_engine and source", + }, + []string{"rule_name", "type", "appsec_engine", "source"}, +) diff --git a/pkg/metrics/acquisition_cloudwatch.go b/pkg/metrics/acquisition_cloudwatch.go new file mode 100644 index 000000000..5b427478d --- /dev/null +++ b/pkg/metrics/acquisition_cloudwatch.go @@ -0,0 +1,32 @@ +//go:build !no_datasource_cloudwatch + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const CloudWatchDatasourceOpenedStreamsMetricName = "cs_cloudwatch_openstreams_total" + +var CloudWatchDatasourceOpenedStreams = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: CloudWatchDatasourceOpenedStreamsMetricName, + Help: "Number of opened stream within group.", + }, + []string{"group"}, +) + +const CloudWatchDatasourceLinesReadMetricName = "cs_cloudwatch_stream_hits_total" + +var CloudWatchDatasourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: CloudWatchDatasourceLinesReadMetricName, + Help: "Number of event read from stream.", + }, + []string{"group", "stream", "datasource_type", "acquis_type"}, +) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(CloudWatchDatasourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_docker.go b/pkg/metrics/acquisition_docker.go new file mode 100644 index 000000000..8e7b98247 --- /dev/null +++ b/pkg/metrics/acquisition_docker.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_docker + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const DockerDatasourceLinesReadMetricName = "cs_dockersource_hits_total" + +var DockerDatasourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: DockerDatasourceLinesReadMetricName, + Help: "Total lines that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(DockerDatasourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_file.go b/pkg/metrics/acquisition_file.go new file mode 100644 index 000000000..1e66d9e26 --- /dev/null +++ b/pkg/metrics/acquisition_file.go @@ -0,0 +1,22 @@ +//go:build !no_datasource_file + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const FileDatasourceLinesReadMetricName = "cs_filesource_hits_total" + +var FileDatasourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: FileDatasourceLinesReadMetricName, + Help: "Total lines that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}, +) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(FileDatasourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_http.go b/pkg/metrics/acquisition_http.go new file mode 100644 index 000000000..7f5a7be1a --- /dev/null +++ b/pkg/metrics/acquisition_http.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_http + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const HTTPDataSourceLinesReadMetricName = "cs_httpsource_hits_total" + +var HTTPDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: HTTPDataSourceLinesReadMetricName, + Help: "Total lines that were read from http source", + }, + []string{"path", "src", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(HTTPDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_journalctl.go b/pkg/metrics/acquisition_journalctl.go new file mode 100644 index 000000000..ce626fdf4 --- /dev/null +++ b/pkg/metrics/acquisition_journalctl.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_journalctl + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const JournalCtlDataSourceLinesReadMetricName = "cs_journalctlsource_hits_total" + +var JournalCtlDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: JournalCtlDataSourceLinesReadMetricName, + Help: "Total lines that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(JournalCtlDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_k8s_audit.go b/pkg/metrics/acquisition_k8s_audit.go new file mode 100644 index 000000000..997dad766 --- /dev/null +++ b/pkg/metrics/acquisition_k8s_audit.go @@ -0,0 +1,30 @@ +//go:build !no_datasource_k8saudit + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const K8SAuditDataSourceEventCountMetricName = "cs_k8sauditsource_hits_total" + +var K8SAuditDataSourceEventCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: K8SAuditDataSourceEventCountMetricName, + Help: "Total number of events received by k8s-audit source", + }, + []string{"source", "datasource_type", "acquis_type"}) + +const K8SAuditDataSourceRequestCountMetricName = "cs_k8sauditsource_requests_total" + +var K8SAuditDataSourceRequestCount = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: K8SAuditDataSourceRequestCountMetricName, + Help: "Total number of requests received", + }, + []string{"source"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(K8SAuditDataSourceEventCountMetricName) +} diff --git a/pkg/metrics/acquisition_kafka.go b/pkg/metrics/acquisition_kafka.go new file mode 100644 index 000000000..8d1bc8d2b --- /dev/null +++ b/pkg/metrics/acquisition_kafka.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_kafka + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const KafkaDataSourceLinesReadMetricName = "cs_kafkasource_hits_total" + +var KafkaDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: KafkaDataSourceLinesReadMetricName, + Help: "Total lines that were read from topic", + }, + []string{"topic", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(KafkaDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_kinesys.go b/pkg/metrics/acquisition_kinesys.go new file mode 100644 index 000000000..27eea9695 --- /dev/null +++ b/pkg/metrics/acquisition_kinesys.go @@ -0,0 +1,32 @@ +//go:build !no_datasource_kinesis + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const KinesisDataSourceLinesReadMetricName = "cs_kinesis_stream_hits_total" + +var KinesisDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: KinesisDataSourceLinesReadMetricName, + Help: "Number of event read per stream.", + }, + []string{"stream", "datasource_type", "acquis_type"}, +) + +const KinesisDataSourceLinesReadShardsMetricName = "cs_kinesis_shards_hits_total" + +var KinesisDataSourceLinesReadShards = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: KinesisDataSourceLinesReadShardsMetricName, + Help: "Number of event read per shards.", + }, + []string{"stream", "shard"}, +) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(KinesisDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_loki.go b/pkg/metrics/acquisition_loki.go new file mode 100644 index 000000000..72cd60446 --- /dev/null +++ b/pkg/metrics/acquisition_loki.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_loki + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const LokiDataSourceLinesReadMetricName = "cs_lokisource_hits_total" + +var LokiDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LokiDataSourceLinesReadMetricName, + Help: "Total lines that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(LokiDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_s3.go b/pkg/metrics/acquisition_s3.go new file mode 100644 index 000000000..3ed6a6c94 --- /dev/null +++ b/pkg/metrics/acquisition_s3.go @@ -0,0 +1,42 @@ +//go:build !no_datasource_s3 + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const S3DataSourceLinesReadMetricName = "cs_s3_hits_total" + +var S3DataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: S3DataSourceLinesReadMetricName, + Help: "Number of events read per bucket.", + }, + []string{"bucket", "datasource_type", "acquis_type"}, +) + +const S3DataSourceObjectsReadMetricName = "cs_s3_objects_total" + +var S3DataSourceObjectsRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: S3DataSourceObjectsReadMetricName, + Help: "Number of objects read per bucket.", + }, + []string{"bucket"}, +) + +const S3DataSourceSQSMessagesReceivedMetricName = "cs_s3_sqs_messages_total" + +var S3DataSourceSQSMessagesReceived = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: S3DataSourceSQSMessagesReceivedMetricName, + Help: "Number of SQS messages received per queue.", + }, + []string{"queue"}, +) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(S3DataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_syslog.go b/pkg/metrics/acquisition_syslog.go new file mode 100644 index 000000000..578857c8e --- /dev/null +++ b/pkg/metrics/acquisition_syslog.go @@ -0,0 +1,30 @@ +//go:build !no_datasource_syslog + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const SyslogDataSourceLinesReceivedMetricName = "cs_syslogsource_hits_total" + +var SyslogDataSourceLinesReceived = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: SyslogDataSourceLinesReceivedMetricName, + Help: "Total lines that were received.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +const SyslogDataSourceLinesParsedMetricName = "cs_syslogsource_parsed_total" + +var SyslogDataSourceLinesParsed = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: SyslogDataSourceLinesParsedMetricName, + Help: "Total lines that were successfully parsed", + }, + []string{"source", "type", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(SyslogDataSourceLinesParsedMetricName) +} diff --git a/pkg/metrics/acquisition_victorialogs.go b/pkg/metrics/acquisition_victorialogs.go new file mode 100644 index 000000000..b85534fe6 --- /dev/null +++ b/pkg/metrics/acquisition_victorialogs.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_victorialogs + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const VictorialogsDataSourceLinesReadMetricName = "cs_victorialogssource_hits_total" + +var VictorialogsDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: VictorialogsDataSourceLinesReadMetricName, + Help: "Total lines that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(VictorialogsDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/acquisition_wineventlog.go b/pkg/metrics/acquisition_wineventlog.go new file mode 100644 index 000000000..aef14bc81 --- /dev/null +++ b/pkg/metrics/acquisition_wineventlog.go @@ -0,0 +1,21 @@ +//go:build !no_datasource_wineventlog + +package metrics + +import ( + "github.com/prometheus/client_golang/prometheus" +) + +const WineventlogDataSourceLinesReadMetricName = "cs_winevtlogsource_hits_total" + +var WineventlogDataSourceLinesRead = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: WineventlogDataSourceLinesReadMetricName, + Help: "Total event that were read.", + }, + []string{"source", "datasource_type", "acquis_type"}) + +//nolint:gochecknoinits +func init() { + RegisterAcquisitionMetric(WineventlogDataSourceLinesReadMetricName) +} diff --git a/pkg/metrics/cache.go b/pkg/metrics/cache.go new file mode 100644 index 000000000..d8520f4da --- /dev/null +++ b/pkg/metrics/cache.go @@ -0,0 +1,13 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const CacheMetricName = "cs_cache_size" + +var CacheMetrics = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: CacheMetricName, + Help: "Entries per cache.", + }, + []string{"name", "type"}, +) diff --git a/pkg/metrics/expr.go b/pkg/metrics/expr.go new file mode 100644 index 000000000..9252ef581 --- /dev/null +++ b/pkg/metrics/expr.go @@ -0,0 +1,13 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const RegexpCacheMetricName = "cs_regexp_cache_size" + +var RegexpCacheMetrics = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: RegexpCacheMetricName, + Help: "Entries per regexp cache.", + }, + []string{"name"}, +) diff --git a/pkg/metrics/global.go b/pkg/metrics/global.go new file mode 100644 index 000000000..c1a60b595 --- /dev/null +++ b/pkg/metrics/global.go @@ -0,0 +1,106 @@ +package metrics + +import ( + "github.com/crowdsecurity/go-cs-lib/version" + "github.com/prometheus/client_golang/prometheus" +) + +const GlobalParserHitsMetricName = "cs_parser_hits_total" + +var GlobalParserHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: GlobalParserHitsMetricName, + Help: "Total events entered the parser.", + }, + []string{"source", "type"}, +) + +const GlobalParserHitsOkMetricName = "cs_parser_hits_ok_total" + +var GlobalParserHitsOk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: GlobalParserHitsOkMetricName, + Help: "Total events were successfully parsed.", + }, + []string{"source", "type", "acquis_type"}, +) + +const GlobalParserHitsKoMetricName = "cs_parser_hits_ko_total" + +var GlobalParserHitsKo = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: GlobalParserHitsKoMetricName, + Help: "Total events were unsuccessfully parsed.", + }, + []string{"source", "type", "acquis_type"}, +) + +const GlobalBucketPourKoMetricName = "cs_bucket_pour_ko_total" + +var GlobalBucketPourKo = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: GlobalBucketPourKoMetricName, + Help: "Total events were not poured in a bucket.", + }, +) + +const GlobalBucketPourOkMetricName = "cs_bucket_pour_ok_total" + +var GlobalBucketPourOk = prometheus.NewCounter( + prometheus.CounterOpts{ + Name: GlobalBucketPourOkMetricName, + Help: "Total events were poured in at least one bucket.", + }, +) + +const GlobalCsInfoMetricName = "cs_info" + +var GlobalCsInfo = prometheus.NewGauge( + prometheus.GaugeOpts{ + Name: GlobalCsInfoMetricName, + Help: "Information about Crowdsec.", + ConstLabels: prometheus.Labels{"version": version.String()}, + }, +) + +const GlobalActiveDecisionsMetricName = "cs_active_decisions" + +var GlobalActiveDecisions = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: GlobalActiveDecisionsMetricName, + Help: "Number of active decisions.", + }, + []string{"reason", "origin", "action"}, +) + +const GlobalAlertsMetricName = "cs_alerts" + +var GlobalAlerts = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: GlobalAlertsMetricName, + Help: "Number of alerts (excluding CAPI).", + }, + []string{"reason"}, +) + +const GlobalParsingHistogramMetricName = "cs_parsing_time_seconds" + +var GlobalParsingHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Help: "Time spent parsing a line", + Name: GlobalParsingHistogramMetricName, + Buckets: []float64{0.0005, 0.001, 0.0015, 0.002, 0.0025, 0.003, 0.004, 0.005, 0.0075, 0.01}, + }, + []string{"type", "source"}, +) + +const GlobalPourHistogramMetricName = "cs_bucket_pour_seconds" + +var GlobalPourHistogram = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: GlobalPourHistogramMetricName, + Help: "Time spent pouring an event to buckets.", + Buckets: []float64{0.001, 0.002, 0.005, 0.01, 0.015, 0.02, 0.03, 0.04, 0.05}, + }, + []string{"type", "source"}, +) diff --git a/pkg/metrics/lapi.go b/pkg/metrics/lapi.go new file mode 100644 index 000000000..c221af26e --- /dev/null +++ b/pkg/metrics/lapi.go @@ -0,0 +1,72 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const LapiRouteHitsMetricName = "cs_lapi_route_requests_total" + +var LapiRouteHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LapiRouteHitsMetricName, + Help: "Number of calls to each route per method.", + }, + []string{"route", "method"}, +) + +/*hits per machine*/ +const LapiMachineHitsMetricName = "cs_lapi_machine_requests_total" + +var LapiMachineHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LapiMachineHitsMetricName, + Help: "Number of calls to each route per method grouped by machines.", + }, + []string{"machine", "route", "method"}, +) + +/*hits per bouncer*/ +const LapiBouncerHitsMetricName = "cs_lapi_bouncer_requests_total" + +var LapiBouncerHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LapiBouncerHitsMetricName, + Help: "Number of calls to each route per method grouped by bouncers.", + }, + []string{"bouncer", "route", "method"}, +) + +/* + keep track of the number of calls (per bouncer) that lead to nil/non-nil responses. + +while it's not exact, it's a good way to know - when you have a rutpure bouncer - what is the rate of ok/ko answers you got from lapi +*/ +const LapiNilDecisionsMetricName = "cs_lapi_decisions_ko_total" + +var LapiNilDecisions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LapiNilDecisionsMetricName, + Help: "Number of calls to /decisions that returned nil result.", + }, + []string{"bouncer"}, +) + +/*hits per bouncer*/ +const LapiNonNilDecisionsMetricName = "cs_lapi_decisions_ok_total" + +var LapiNonNilDecisions = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LapiNonNilDecisionsMetricName, + Help: "Number of calls to /decisions that returned non-nil result.", + }, + []string{"bouncer"}, +) + +const LapiResponseTimeMetricName = "cs_lapi_request_duration_seconds" + +var LapiResponseTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: LapiResponseTimeMetricName, + Help: "Response time of LAPI", + Buckets: []float64{0.005, 0.01, 0.025, 0.05, 0.075, 0.1, 0.2, 0.3, 0.4, 0.5, 0.75, 1}, + }, + []string{"endpoint", "method"}, +) diff --git a/pkg/metrics/leaky.go b/pkg/metrics/leaky.go new file mode 100644 index 000000000..68ad47aee --- /dev/null +++ b/pkg/metrics/leaky.go @@ -0,0 +1,63 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const BucketPouredMetricName = "cs_bucket_poured_total" + +var BucketsPour = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: BucketPouredMetricName, + Help: "Total events were poured in bucket.", + }, + []string{"source", "type", "name"}, +) + +const BucketsOverflowMetricName = "cs_bucket_overflowed_total" + +var BucketsOverflow = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: BucketsOverflowMetricName, + Help: "Total buckets overflowed.", + }, + []string{"name"}, +) + +const BucketsCanceledMetricName = "cs_bucket_canceled_total" + +var BucketsCanceled = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: BucketsCanceledMetricName, + Help: "Total buckets canceled.", + }, + []string{"name"}, +) + +const BucketsUnderflowMetricName = "cs_bucket_underflowed_total" + +var BucketsUnderflow = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: BucketsUnderflowMetricName, + Help: "Total buckets underflowed.", + }, + []string{"name"}, +) + +const BucketsInstantiationMetricName = "cs_bucket_instantiation_total" + +var BucketsInstantiation = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: BucketsInstantiationMetricName, + Help: "Total buckets were instantiated.", + }, + []string{"name"}, +) + +const BucketsCurrentCountMetricName = "cs_buckets" + +var BucketsCurrentCount = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Name: BucketsCurrentCountMetricName, + Help: "Number of buckets that currently exist.", + }, + []string{"name"}, +) diff --git a/pkg/metrics/metrics.go b/pkg/metrics/metrics.go new file mode 100644 index 000000000..5da481170 --- /dev/null +++ b/pkg/metrics/metrics.go @@ -0,0 +1,54 @@ +package metrics + +import ( + "errors" + "fmt" + + "github.com/prometheus/client_golang/prometheus" +) + +type MetricsLevelConfig string + +const ( + MetricsLevelNone MetricsLevelConfig = "none" + MetricsLevelAggregated MetricsLevelConfig = "aggregated" + MetricsLevelFull MetricsLevelConfig = "full" + // MetricsLevelDefault is the default metrics level. + MetricsLevelDefault MetricsLevelConfig = MetricsLevelFull +) + +var ErrInvalidMetricsLevel = errors.New("invalid metrics level") + +type AcquisitionMetricsLevel int + +const ( + AcquisitionMetricsLevelNone AcquisitionMetricsLevel = iota // No metrics + AcquisitionMetricsLevelAggregated // Aggregated metrics + AcquisitionMetricsLevelFull // Full metrics + AcquisitionMetricsLevelDefault = AcquisitionMetricsLevelFull // Default metrics level +) + +func RegisterMetrics(metricsLevel MetricsLevelConfig) error { + switch metricsLevel { + case MetricsLevelNone: + // Do not register any metrics + case MetricsLevelAggregated: + prometheus.MustRegister(GlobalParserHits, GlobalParserHitsOk, GlobalParserHitsKo, + GlobalCsInfo, GlobalParsingHistogram, GlobalPourHistogram, + BucketsUnderflow, BucketsCanceled, BucketsInstantiation, BucketsOverflow, + LapiRouteHits, + BucketsCurrentCount, + CacheMetrics, RegexpCacheMetrics, NodesWlHitsOk, NodesWlHits) + case MetricsLevelFull: + prometheus.MustRegister(GlobalParserHits, GlobalParserHitsOk, GlobalParserHitsKo, + NodesHits, NodesHitsOk, NodesHitsKo, + GlobalCsInfo, GlobalParsingHistogram, GlobalPourHistogram, + LapiRouteHits, LapiMachineHits, LapiBouncerHits, LapiNilDecisions, LapiNonNilDecisions, LapiResponseTime, + BucketsPour, BucketsUnderflow, BucketsCanceled, BucketsInstantiation, BucketsOverflow, BucketsCurrentCount, + GlobalActiveDecisions, GlobalAlerts, NodesWlHitsOk, NodesWlHits, + CacheMetrics, RegexpCacheMetrics) + default: + return fmt.Errorf("%w: %s", ErrInvalidMetricsLevel, metricsLevel) + } + return nil +} diff --git a/pkg/metrics/parser.go b/pkg/metrics/parser.go new file mode 100644 index 000000000..3e99ca6d2 --- /dev/null +++ b/pkg/metrics/parser.go @@ -0,0 +1,53 @@ +package metrics + +import "github.com/prometheus/client_golang/prometheus" + +const NodesHitsMetricName = "cs_node_hits_total" + +var NodesHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: NodesHitsMetricName, + Help: "Total events entered node.", + }, + []string{"source", "type", "name", "stage", "acquis_type"}, +) + +const NodesHitsOkMetricName = "cs_node_hits_ok_total" + +var NodesHitsOk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: NodesHitsOkMetricName, + Help: "Total events successfully exited node.", + }, + []string{"source", "type", "name", "stage", "acquis_type"}, +) + +const NodesHitsKoMetricName = "cs_node_hits_ko_total" + +var NodesHitsKo = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: NodesHitsKoMetricName, + Help: "Total events unsuccessfully exited node.", + }, + []string{"source", "type", "name", "stage", "acquis_type"}, +) + +const NodesWlHitsOkMetricName = "cs_node_wl_hits_ok_total" + +var NodesWlHitsOk = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: NodesWlHitsOkMetricName, + Help: "Total events successfully whitelisted by node.", + }, + []string{"source", "type", "name", "reason", "stage", "acquis_type"}, +) + +const NodesWlHitsMetricName = "cs_node_wl_hits_total" + +var NodesWlHits = prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: NodesWlHitsMetricName, + Help: "Total events processed by whitelist node.", + }, + []string{"source", "type", "name", "reason", "stage", "acquis_type"}, +) diff --git a/pkg/parser/node.go b/pkg/parser/node.go index 7332356b9..4ef3075b3 100644 --- a/pkg/parser/node.go +++ b/pkg/parser/node.go @@ -19,6 +19,7 @@ import ( "github.com/crowdsecurity/crowdsec/pkg/cache" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -297,7 +298,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } if n.Name != "" { - NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() + metrics.NodesHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() } isWhitelisted, err := n.processWhitelist(cachedExprEnv, p) @@ -396,7 +397,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri // grok or leafs failed, don't process statics if !NodeState { if n.Name != "" { - NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() + metrics.NodesHitsKo.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() } clog.Debugf("Event leaving node : ko") @@ -405,7 +406,7 @@ func (n *Node) process(p *types.Event, ctx UnixParserCtx, expressionEnv map[stri } if n.Name != "" { - NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name}).Inc() + metrics.NodesHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() } /* diff --git a/pkg/parser/runtime.go b/pkg/parser/runtime.go index 07e7a4a98..2b9e86701 100644 --- a/pkg/parser/runtime.go +++ b/pkg/parser/runtime.go @@ -15,7 +15,6 @@ import ( "time" "github.com/mohae/deepcopy" - "github.com/prometheus/client_golang/prometheus" log "github.com/sirupsen/logrus" "github.com/crowdsecurity/crowdsec/pkg/dumps" @@ -210,48 +209,6 @@ func (n *Node) ProcessStatics(statics []ExtraField, event *types.Event) error { return nil } -var NodesHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_total", - Help: "Total events entered node.", - }, - []string{"source", "type", "name"}, -) - -var NodesHitsOk = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_ok_total", - Help: "Total events successfully exited node.", - }, - []string{"source", "type", "name"}, -) - -var NodesHitsKo = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_hits_ko_total", - Help: "Total events unsuccessfully exited node.", - }, - []string{"source", "type", "name"}, -) - -// - -var NodesWlHitsOk = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_wl_hits_ok_total", - Help: "Total events successfully whitelisted by node.", - }, - []string{"source", "type", "name", "reason"}, -) - -var NodesWlHits = prometheus.NewCounterVec( - prometheus.CounterOpts{ - Name: "cs_node_wl_hits_total", - Help: "Total events processed by whitelist node.", - }, - []string{"source", "type", "name", "reason"}, -) - func stageidx(stage string, stages []string) int { for i, v := range stages { if stage == v { diff --git a/pkg/parser/whitelist.go b/pkg/parser/whitelist.go index e7b93a8d7..491d74c9a 100644 --- a/pkg/parser/whitelist.go +++ b/pkg/parser/whitelist.go @@ -9,6 +9,7 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/crowdsecurity/crowdsec/pkg/exprhelpers" + "github.com/crowdsecurity/crowdsec/pkg/metrics" "github.com/crowdsecurity/crowdsec/pkg/types" ) @@ -44,7 +45,8 @@ func (n *Node) CheckIPsWL(p *types.Event) bool { if !n.ContainsIPLists() { return isWhitelisted } - NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc() + metrics.NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason, + "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() for _, src := range srcs { if isWhitelisted { break @@ -67,7 +69,8 @@ func (n *Node) CheckIPsWL(p *types.Event) bool { } } if isWhitelisted { - NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc() + metrics.NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason, + "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() } return isWhitelisted } @@ -78,7 +81,8 @@ func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}, p *types.Event) if !n.ContainsExprLists() { return false, nil } - NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc() + metrics.NodesWlHits.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason, + "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() /* run whitelist expression tests anyway */ for eidx, e := range n.Whitelist.B_Exprs { //if we already know the event is whitelisted, skip the rest of the expressions @@ -103,7 +107,8 @@ func (n *Node) CheckExprWL(cachedExprEnv map[string]interface{}, p *types.Event) } } if isWhitelisted { - NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason}).Inc() + metrics.NodesWlHitsOk.With(prometheus.Labels{"source": p.Line.Src, "type": p.Line.Module, "name": n.Name, "reason": n.Whitelist.Reason, + "stage": p.Stage, "acquis_type": p.Line.Labels["type"]}).Inc() } return isWhitelisted, nil }