[receiver/prometheus] Support OpenMetrics Info and Stateset metrics (#…

…9378)

* [receiver/prometheus] support OpenMetrics Info and Stateset metrics

* Update CHANGELOG.md

* Update receiver/prometheusreceiver/internal/otlp_metricsbuilder.go

Co-authored-by: Alex Boten <aboten@lightstep.com>

CHANGELOG.md

@@ -117,6 +117,7 @@
 - `transformprocessor`: Add new `limit` function to allow limiting the number of items in a map, such as the number of attributes in `attributes` or `resource.attributes` (#9552)
 - `processor/attributes`: Support attributes set by server authenticator (#9420)
 - `datadogexporter`: Experimental support for Exponential Histograms with delta aggregation temporality (#8350)
+- `prometheusreceiver`: Support OpenMetrics Info and Stateset metrics (#9378)
 
 ### 🧰 Bug fixes 🧰
 

receiver/prometheusreceiver/internal/otlp_metricfamily.go

@@ -29,7 +29,9 @@ import (
 )
 
 type metricFamily struct {
-	mtype             pmetric.MetricDataType
+	mtype pmetric.MetricDataType
+	// isMonotonic only applies to sums
+	isMonotonic       bool
 	groups            map[string]*metricGroup
 	name              string
 	mc                MetadataCache
@@ -59,13 +61,14 @@ var pdataStaleFlags = pmetric.NewMetricDataPointFlags(pmetric.MetricDataPointFla
 
 func newMetricFamily(metricName string, mc MetadataCache, logger *zap.Logger) *metricFamily {
 	metadata, familyName := metadataForMetric(metricName, mc)
-	mtype := convToMetricType(metadata.Type)
+	mtype, isMonotonic := convToMetricType(metadata.Type)
 	if mtype == pmetric.MetricDataTypeNone {
 		logger.Debug(fmt.Sprintf("Unknown-typed metric : %s %+v", metricName, metadata))
 	}
 
 	return &metricFamily{
 		mtype:             mtype,
+		isMonotonic:       isMonotonic,
 		groups:            make(map[string]*metricGroup),
 		name:              familyName,
 		mc:                mc,
@@ -354,7 +357,7 @@ func (mf *metricFamily) ToMetric(metrics *pmetric.MetricSlice) (int, int) {
 	case pmetric.MetricDataTypeSum:
 		sum := metric.Sum()
 		sum.SetAggregationTemporality(pmetric.MetricAggregationTemporalityCumulative)
-		sum.SetIsMonotonic(true)
+		sum.SetIsMonotonic(mf.isMonotonic)
 		sdpL := sum.DataPoints()
 		for _, mg := range mf.getGroups() {
 			if !mg.toNumberDataPoint(mf.labelKeysOrdered, &sdpL) {

receiver/prometheusreceiver/internal/otlp_metricsbuilder.go

@@ -59,25 +59,28 @@ func getBoundary(metricType pmetric.MetricDataType, labels labels.Labels) (float
 	return strconv.ParseFloat(v, 64)
 }
 
-func convToMetricType(metricType textparse.MetricType) pmetric.MetricDataType {
+// convToMetricType returns the data type and if it is monotonic
+func convToMetricType(metricType textparse.MetricType) (pmetric.MetricDataType, bool) {
 	switch metricType {
 	case textparse.MetricTypeCounter:
 		// always use float64, as it's the internal data type used in prometheus
-		return pmetric.MetricDataTypeSum
+		return pmetric.MetricDataTypeSum, true
 	// textparse.MetricTypeUnknown is converted to gauge by default to prevent Prometheus untyped metrics from being dropped
 	case textparse.MetricTypeGauge, textparse.MetricTypeUnknown:
-		return pmetric.MetricDataTypeGauge
+		return pmetric.MetricDataTypeGauge, false
 	case textparse.MetricTypeHistogram:
-		return pmetric.MetricDataTypeHistogram
+		return pmetric.MetricDataTypeHistogram, true
 	// dropping support for gaugehistogram for now until we have an official spec of its implementation
 	// a draft can be found in: https://docs.google.com/document/d/1KwV0mAXwwbvvifBvDKH_LU1YjyXE_wxCkHNoCGq1GX0/edit#heading=h.1cvzqd4ksd23
 	// case textparse.MetricTypeGaugeHistogram:
 	//	return <pdata gauge histogram type>
 	case textparse.MetricTypeSummary:
-		return pmetric.MetricDataTypeSummary
+		return pmetric.MetricDataTypeSummary, true
+	case textparse.MetricTypeInfo, textparse.MetricTypeStateset:
+		return pmetric.MetricDataTypeSum, false
 	default:
-		// including: textparse.MetricTypeGaugeHistogram, textparse.MetricTypeInfo, textparse.MetricTypeStateset
-		return pmetric.MetricDataTypeNone
+		// including: textparse.MetricTypeGaugeHistogram
+		return pmetric.MetricDataTypeNone, false
 	}
 }
 

receiver/prometheusreceiver/internal/otlp_metricsbuilder_test.go

@@ -182,52 +182,67 @@ func TestGetBoundary(t *testing.T) {
 
 func TestConvToMetricType(t *testing.T) {
 	tests := []struct {
-		name  string
-		mtype textparse.MetricType
-		want  pmetric.MetricDataType
+		name          string
+		mtype         textparse.MetricType
+		want          pmetric.MetricDataType
+		wantMonotonic bool
 	}{
 		{
-			name:  "textparse.counter",
-			mtype: textparse.MetricTypeCounter,
-			want:  pmetric.MetricDataTypeSum,
+			name:          "textparse.counter",
+			mtype:         textparse.MetricTypeCounter,
+			want:          pmetric.MetricDataTypeSum,
+			wantMonotonic: true,
 		},
 		{
-			name:  "textparse.gauge",
-			mtype: textparse.MetricTypeGauge,
-			want:  pmetric.MetricDataTypeGauge,
+			name:          "textparse.gauge",
+			mtype:         textparse.MetricTypeGauge,
+			want:          pmetric.MetricDataTypeGauge,
+			wantMonotonic: false,
 		},
 		{
-			name:  "textparse.unknown",
-			mtype: textparse.MetricTypeUnknown,
-			want:  pmetric.MetricDataTypeGauge,
+			name:          "textparse.unknown",
+			mtype:         textparse.MetricTypeUnknown,
+			want:          pmetric.MetricDataTypeGauge,
+			wantMonotonic: false,
 		},
 		{
-			name:  "textparse.histogram",
-			mtype: textparse.MetricTypeHistogram,
-			want:  pmetric.MetricDataTypeHistogram,
+			name:          "textparse.histogram",
+			mtype:         textparse.MetricTypeHistogram,
+			want:          pmetric.MetricDataTypeHistogram,
+			wantMonotonic: true,
 		},
 		{
-			name:  "textparse.summary",
-			mtype: textparse.MetricTypeSummary,
-			want:  pmetric.MetricDataTypeSummary,
+			name:          "textparse.summary",
+			mtype:         textparse.MetricTypeSummary,
+			want:          pmetric.MetricDataTypeSummary,
+			wantMonotonic: true,
 		},
 		{
-			name:  "textparse.metric_type_info",
-			mtype: textparse.MetricTypeInfo,
-			want:  pmetric.MetricDataTypeNone,
+			name:          "textparse.metric_type_info",
+			mtype:         textparse.MetricTypeInfo,
+			want:          pmetric.MetricDataTypeSum,
+			wantMonotonic: false,
 		},
 		{
-			name:  "textparse.metric_state_set",
-			mtype: textparse.MetricTypeStateset,
-			want:  pmetric.MetricDataTypeNone,
+			name:          "textparse.metric_state_set",
+			mtype:         textparse.MetricTypeStateset,
+			want:          pmetric.MetricDataTypeSum,
+			wantMonotonic: false,
+		},
+		{
+			name:          "textparse.metric_gauge_hostogram",
+			mtype:         textparse.MetricTypeGaugeHistogram,
+			want:          pmetric.MetricDataTypeNone,
+			wantMonotonic: false,
 		},
 	}
 
 	for _, tt := range tests {
 		tt := tt
 		t.Run(tt.name, func(t *testing.T) {
-			got := convToMetricType(tt.mtype)
+			got, monotonic := convToMetricType(tt.mtype)
 			require.Equal(t, got.String(), tt.want.String())
+			require.Equal(t, monotonic, tt.wantMonotonic)
 		})
 	}
 }

receiver/prometheusreceiver/internal/util.go

@@ -27,6 +27,7 @@ const (
 	metricsSuffixBucket = "_bucket"
 	metricsSuffixSum    = "_sum"
 	metricSuffixTotal   = "_total"
+	metricSuffixInfo    = "_info"
 	startTimeMetricName = "process_start_time_seconds"
 	scrapeUpMetricName  = "up"
 
@@ -35,7 +36,7 @@ const (
 )
 
 var (
-	trimmableSuffixes     = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal}
+	trimmableSuffixes     = []string{metricsSuffixBucket, metricsSuffixCount, metricsSuffixSum, metricSuffixTotal, metricSuffixInfo}
 	errNoDataToBuild      = errors.New("there's no data to build")
 	errNoBoundaryLabel    = errors.New("given metricType has no BucketLabel or QuantileLabel")
 	errEmptyBoundaryLabel = errors.New("BucketLabel or QuantileLabel is empty")

receiver/prometheusreceiver/metrics_receiver_helper_test.go

@@ -395,6 +395,13 @@ func compareMetricType(typ pmetric.MetricDataType) metricTypeComparator {
 	}
 }
 
+func compareMetricIsMonotonic(isMonotonic bool) metricTypeComparator {
+	return func(t *testing.T, metric *pmetric.Metric) {
+		assert.Equal(t, pmetric.MetricDataTypeSum.String(), metric.DataType().String(), "IsMonotonic only exists for sums")
+		assert.Equal(t, isMonotonic, metric.Sum().IsMonotonic(), "IsMonotonic does not match")
+	}
+}
+
 func compareAttributes(attributes map[string]string) numberPointComparator {
 	return func(t *testing.T, numberDataPoint *pmetric.NumberDataPoint) {
 		req := assert.Equal(t, len(attributes), numberDataPoint.Attributes().Len(), "Attributes length do not match")

receiver/prometheusreceiver/metrics_receiver_open_metrics_test.go

@@ -24,6 +24,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 )
@@ -156,3 +157,115 @@ func readTestCase(testName string) (string, error) {
 	}
 	return string(content), nil
 }
+
+// info and stateset metrics are converted to non-monotonic sums
+var infoAndStatesetMetrics = `# TYPE foo info
+foo_info{entity="controller",name="prettyname",version="8.2.7"} 1.0
+foo_info{entity="replica",name="prettiername",version="8.1.9"} 1.0
+# TYPE bar stateset
+bar{entity="controller",foo="a"} 1.0
+bar{entity="controller",foo="bb"} 0.0
+bar{entity="controller",foo="ccc"} 0.0
+bar{entity="replica",foo="a"} 1.0
+bar{entity="replica",foo="bb"} 0.0
+bar{entity="replica",foo="ccc"} 1.0
+# EOF
+`
+
+// TestInfoStatesetMetrics validates the translation of info and stateset
+// metrics
+func TestInfoStatesetMetrics(t *testing.T) {
+	targets := []*testData{
+		{
+			name: "target1",
+			pages: []mockPrometheusResponse{
+				{code: 200, data: infoAndStatesetMetrics, useOpenMetrics: true},
+			},
+			validateFunc:    verifyInfoStatesetMetrics,
+			validateScrapes: true,
+		},
+	}
+
+	testComponent(t, targets, false, "")
+
+}
+
+func verifyInfoStatesetMetrics(t *testing.T, td *testData, resourceMetrics []*pmetric.ResourceMetrics) {
+	verifyNumValidScrapeResults(t, td, resourceMetrics)
+	m1 := resourceMetrics[0]
+
+	// m1 has 2 metrics + 5 internal scraper metrics
+	assert.Equal(t, 7, metricsCount(m1))
+
+	wantAttributes := td.attributes
+
+	metrics1 := m1.ScopeMetrics().At(0).Metrics()
+	ts1 := getTS(metrics1)
+	e1 := []testExpectation{
+		assertMetricPresent("foo",
+			compareMetricIsMonotonic(false),
+			[]dataPointExpectation{
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(1.0),
+						compareAttributes(map[string]string{"entity": "controller", "name": "prettyname", "version": "8.2.7"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(1.0),
+						compareAttributes(map[string]string{"entity": "replica", "name": "prettiername", "version": "8.1.9"}),
+					},
+				},
+			}),
+		assertMetricPresent("bar",
+			compareMetricIsMonotonic(false),
+			[]dataPointExpectation{
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(1.0),
+						compareAttributes(map[string]string{"entity": "controller", "foo": "a"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(0.0),
+						compareAttributes(map[string]string{"entity": "controller", "foo": "bb"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(0.0),
+						compareAttributes(map[string]string{"entity": "controller", "foo": "ccc"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(1.0),
+						compareAttributes(map[string]string{"entity": "replica", "foo": "a"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(0.0),
+						compareAttributes(map[string]string{"entity": "replica", "foo": "bb"}),
+					},
+				},
+				{
+					numberPointComparator: []numberPointComparator{
+						compareTimestamp(ts1),
+						compareDoubleValue(1.0),
+						compareAttributes(map[string]string{"entity": "replica", "foo": "ccc"}),
+					},
+				},
+			}),
+	}
+	doCompare(t, "scrape-infostatesetmetrics-1", wantAttributes, m1, e1)
+}