Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

feat(metricprovider): add support for specifying a datasource for a given query #3560

Open
wants to merge 7 commits into
base: master
Choose a base branch
from
Open

feat(metricprovider): add support for specifying a datasource for a given query #3560

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
5d9b61d
feat(datadog): add support for specifying a datasource for a given qu…
backjo May 7, 2024
5ff813d
add test case
backjo May 8, 2024
8674989
add test case
backjo May 8, 2024
fbd8a7f
add test case
backjo May 8, 2024
a408a7c
add test case
backjo May 8, 2024
81c7a3a
fix formatting
backjo May 9, 2024
b0bc0bf
fix formatting
backjo May 9, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 18 additions & 0 deletions docs/features/kustomize/rollout_cr_schema.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,12 @@
],
"type": "string"
},
"datasources": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"formula": {
"type": "string"
},
Expand Down Expand Up @@ -5104,6 +5110,12 @@
],
"type": "string"
},
"datasources": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"formula": {
"type": "string"
},
Expand Down Expand Up @@ -9955,6 +9967,12 @@
],
"type": "string"
},
"datasources": {
"additionalProperties": {
"type": "string"
},
"type": "object"
},
"formula": {
"type": "string"
},
Expand Down
4 changes: 4 additions & 0 deletions manifests/crds/analysis-run-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -197,6 +197,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down
4 changes: 4 additions & 0 deletions manifests/crds/analysis-template-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down
4 changes: 4 additions & 0 deletions manifests/crds/cluster-analysis-template-crd.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -193,6 +193,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down
12 changes: 12 additions & 0 deletions manifests/install.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,6 +198,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down Expand Up @@ -3488,6 +3492,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down Expand Up @@ -6656,6 +6664,10 @@ spec:
- v1
- v2
type: string
datasources:
additionalProperties:
type: string
type: object
formula:
type: string
interval:
Expand Down
17 changes: 12 additions & 5 deletions metricproviders/datadog/datadog.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,8 +187,11 @@ func (p *Provider) createRequest(dd *v1alpha1.DatadogMetric, now int64, interval
if dd.Query != "" {
dd.Queries = map[string]string{"query": dd.Query}
}
if dd.Datasources == nil {
dd.Datasources = make(map[string]string)
}

return p.createRequestV2(dd.Queries, dd.Formula, now, interval, dd.Aggregator, url)
return p.createRequestV2(dd.Queries, dd.Formula, now, interval, dd.Aggregator, url, dd.Datasources)
}

func (p *Provider) createRequestV1(query string, now int64, interval int64, url *url.URL) (*http.Request, error) {
Expand All @@ -201,12 +204,16 @@ func (p *Provider) createRequestV1(query string, now int64, interval int64, url
return &http.Request{Method: "GET"}, nil
}

func buildQueriesPayload(queries map[string]string, aggregator string) []map[string]string {
func buildQueriesPayload(queries map[string]string, aggregator string, datasources map[string]string) []map[string]string {
qp := make([]map[string]string, 0, len(queries))
for k, v := range queries {
datasource := "metrics"
if datasources[k] != "" {
datasource = datasources[k]
}
p := map[string]string{
"aggregator": aggregator,
"data_source": "metrics",
"data_source": datasource,
"name": k,
"query": v,
}
Expand All @@ -215,7 +222,7 @@ func buildQueriesPayload(queries map[string]string, aggregator string) []map[str
return qp
}

func (p *Provider) createRequestV2(queries map[string]string, formula string, now int64, interval int64, aggregator string, url *url.URL) (*http.Request, error) {
func (p *Provider) createRequestV2(queries map[string]string, formula string, now int64, interval int64, aggregator string, url *url.URL, datasources map[string]string) (*http.Request, error) {
formulas := []map[string]string{}
// ddAPI supports multiple formulas but doesn't make sense in our context
// can't have a 'blank' formula, so have to guard
Expand All @@ -229,7 +236,7 @@ func (p *Provider) createRequestV2(queries map[string]string, formula string, no
// Datadog requires milliseconds for v2 api
From: (now - interval) * 1000,
To: now * 1000,
Queries: buildQueriesPayload(queries, aggregator),
Queries: buildQueriesPayload(queries, aggregator, datasources),
Formulas: formulas,
}

Expand Down
51 changes: 51 additions & 0 deletions metricproviders/datadog/datadogV2_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,24 @@ func newQueryProviderInterval10m() v1alpha1.MetricProvider {
}
}

func newQueriesDefaultProviderDataSources() v1alpha1.MetricProvider {
return v1alpha1.MetricProvider{
Datadog: &v1alpha1.DatadogMetric{
Interval: "5m",
Queries: map[string]string{
"a": "avg:error_requests{*}",
"b": "source:deployment_analysis",
},
Datasources: map[string]string{
"a": "events",
"b": "events",
},
Formula: "a/b",
ApiVersion: "v2",
},
}
}

func TestRunSuiteV2(t *testing.T) {
const expectedApiKey = "0123456789abcdef0123456789abcdef"
const expectedAppKey = "0123456789abcdef0123456789abcdef01234567"
Expand All @@ -66,6 +84,7 @@ func TestRunSuiteV2(t *testing.T) {
metric v1alpha1.Metric
expectedIntervalSeconds int64
expectedValue string
expectedDatasource string
expectedPhase v1alpha1.AnalysisPhase
expectedErrorMessage string
useEnvVarForKeys bool
Expand All @@ -82,6 +101,7 @@ func TestRunSuiteV2(t *testing.T) {
expectedIntervalSeconds: 600,
expectedValue: "0.0006332881882246533",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
expectedDatasource: "metrics",
useEnvVarForKeys: false,
},
{
Expand All @@ -95,6 +115,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 600,
expectedValue: "0.0003332881882246533",
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: true,
},
Expand All @@ -109,6 +130,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: "0.006121374442186943",
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseFailed,
useEnvVarForKeys: false,
},
Expand All @@ -123,6 +145,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedDatasource: "metrics",
expectedErrorMessage: "received non 2xx response code: 400 {\"status\":\"error\",\"error\":\"error messsage\"}",
useEnvVarForKeys: false,
},
Expand All @@ -137,6 +160,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedDatasource: "metrics",
expectedErrorMessage: "received authentication error response code: 401 {\"errors\": [\"No authenticated user.\"]}",
useEnvVarForKeys: false,
},
Expand All @@ -151,6 +175,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: "0.006121378742186943",
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
Expand All @@ -165,6 +190,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedDatasource: "metrics",
expectedErrorMessage: `invalid operation: < (mismatched types <nil> and float64)`,
useEnvVarForKeys: false,
},
Expand All @@ -179,6 +205,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: `[]`,
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
Expand All @@ -193,6 +220,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: `[]`,
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseFailed,
useEnvVarForKeys: false,
},
Expand All @@ -207,6 +235,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: `0.006721378742186999`,
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
Expand All @@ -222,6 +251,7 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedDatasource: "metrics",
expectedErrorMessage: "Could not parse JSON body: json: cannot unmarshal string into Go struct field .Data.Attributes.Columns.Values of type []float64",
useEnvVarForKeys: false,
},
Expand All @@ -233,6 +263,7 @@ func TestRunSuiteV2(t *testing.T) {
Provider: newQueryProviderInterval10m(),
},
expectedPhase: v1alpha1.AnalysisPhaseError,
expectedDatasource: "metrics",
expectedErrorMessage: "parse \"://wrong.schema\": missing protocol scheme",
useEnvVarForKeys: false,
},
Expand All @@ -249,6 +280,21 @@ func TestRunSuiteV2(t *testing.T) {
},
expectedIntervalSeconds: 300,
expectedValue: "0.0006444881882246533",
expectedDatasource: "metrics",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
{
webServerStatus: 200,
webServerResponse: `{"data": {"attributes": {"columns": [ {"values": [0.0006444881882246533]}]}}}`,
metric: v1alpha1.Metric{
Name: "expect success queries and formula",
SuccessCondition: "default(result, 0) < 0.05",
Provider: newQueriesDefaultProviderDataSources(),
},
expectedIntervalSeconds: 300,
expectedValue: "0.0006444881882246533",
expectedDatasource: "events",
expectedPhase: v1alpha1.AnalysisPhaseSuccessful,
useEnvVarForKeys: false,
},
Expand Down Expand Up @@ -280,6 +326,7 @@ func TestRunSuiteV2(t *testing.T) {

actualFormulas := reqBody.Data.Attributes.Formulas
actualQuery := reqBody.Data.Attributes.Queries[0]["query"]
actualDataSource := reqBody.Data.Attributes.Queries[0]["data_source"]
actualQueries := reqBody.Data.Attributes.Queries
actualFrom := reqBody.Data.Attributes.From
actualTo := reqBody.Data.Attributes.To
Expand All @@ -299,6 +346,10 @@ func TestRunSuiteV2(t *testing.T) {
}
}

if actualDataSource != test.expectedDatasource {
t.Errorf("\ndatasource %s expected be equal to %s", actualDataSource, test.expectedDatasource)
}

if actualFrom != (unixNow()-test.expectedIntervalSeconds)*1000 {
t.Errorf("\nfrom %d expected be equal to %d", actualFrom, (unixNow()-test.expectedIntervalSeconds)*1000)
} else if err != nil {
Expand Down
7 changes: 7 additions & 0 deletions pkg/apiclient/rollout/rollout.swagger.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1206,6 +1206,13 @@
"aggregator": {
"type": "string",
"title": "+kubebuilder:default=\"last\"\n+kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area\nAggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2"
},
"datasource": {
"type": "object",
"additionalProperties": {
"type": "string"
},
"title": "+kubebuilder:validation:Type=object"
}
}
},
Expand Down
2 changes: 2 additions & 0 deletions pkg/apis/rollouts/v1alpha1/analysis_types.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -605,4 +605,6 @@ type DatadogMetric struct {
// +kubebuilder:validation:Enum=avg;min;max;sum;last;percentile;mean;l2norm;area
// Aggregator is a type of aggregator to use for metrics-based queries (default: last). Used for v2
Aggregator string `json:"aggregator,omitempty" protobuf:"bytes,6,opt,name=aggregator"`
// +kubebuilder:validation:Type=object
Datasources map[string]string `json:"datasources,omitempty" protobuf:"bytes,7,opt,name=datasource"`
}
Loading
Loading