From 54746b6e254ac9f777ed73503e2c3260f811283f Mon Sep 17 00:00:00 2001 From: Julian Gutierrez Oschmann Date: Fri, 29 Aug 2025 21:24:52 +0000 Subject: [PATCH] feat: Allow to override metrics' host project This change allows to specify the host project of a metric in the metric spec, so that the adapter can fetch metrics from a different project than the one where the adapter is running. This is useful in scenarios where metrics are stored in a central project, but the HPA is running in a different project. The host project can be specified by adding the `resource.labels.metrics_host_project_id` label to the metric selector in the HPA definition. For external metrics, the legacy `resource.labels.project_id` label is also supported for backward compatibility. How to use: To fetch a metric from a different project, add the `resource.labels.metrics_host_project_id` label to the metric selector in the HPA definition. Example for a custom metric: ```yaml apiVersion: autoscaling/v2beta2 kind: HorizontalPodAutoscaler metadata: name: custom-metric-hpa spec: scaleTargetRef: apiVersion: apps/v1 kind: Deployment name: my-app minReplicas: 1 maxReplicas: 10 metrics: - type: Pods pods: metric: name: my-metric selector: matchLabels: "resource.labels.metrics_host_project_id": "my-metrics-project" target: type: AverageValue averageValue: 10 ``` --- .../docs/cross-project.md | 108 +++++++++ .../pkg/adapter/provider/provider.go | 40 +++- .../pkg/adapter/translator/query_builder.go | 176 +++++++-------- .../adapter/translator/query_builder_test.go | 205 ++++++++++++++++-- 4 files changed, 410 insertions(+), 119 deletions(-) create mode 100644 custom-metrics-stackdriver-adapter/docs/cross-project.md diff --git a/custom-metrics-stackdriver-adapter/docs/cross-project.md b/custom-metrics-stackdriver-adapter/docs/cross-project.md new file mode 100644 index 000000000..a3d76aec8 --- /dev/null +++ b/custom-metrics-stackdriver-adapter/docs/cross-project.md @@ -0,0 +1,108 @@ +# Fetching Metrics from a Different Google Cloud Project + +This adapter supports fetching metrics from a Google Cloud project different from the one where the adapter is running. This is useful in scenarios where metrics are stored in a central project, but the Horizontal Pod Autoscaler (HPA) is defined in a cluster running in a different project. + +## Configuration + +To fetch a metric from a different project, you need to specify the target project ID using the label `resource.labels.metrics_host_project_id` within the metric selector in your HPA definition. This applies to all supported metric types: `Pods`, `Object`, and `External`. + +The label should be added to the `matchLabels` section of the `metric.selector` . + +### Examples + +Below are examples for each metric type: + +**1. Pods Metric:** + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: pods-metric-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Pods + pods: + metric: + name: my-pod-metric + selector: + matchLabels: + "resource.labels.metrics_host_project_id": "my-metrics-project" + target: + type: AverageValue + averageValue: 10 +``` + +**2. Object Metric:** + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: object-metric-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: Object + object: + metric: + name: my-object-metric + selector: + matchLabels: + "resource.labels.metrics_host_project_id": "my-metrics-project" + describedObject: + apiVersion: v1 + kind: Service + name: my-service + target: + type: Value + value: 100 +``` + +**3. External Metric:** + +```yaml +apiVersion: autoscaling/v2 +kind: HorizontalPodAutoscaler +metadata: + name: external-metric-hpa +spec: + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app + minReplicas: 1 + maxReplicas: 10 + metrics: + - type: External + external: + metric: + name: my-external-metric + selector: + matchLabels: + "resource.labels.metrics_host_project_id": "my-metrics-project" + target: + type: Value + value: 10 +``` + +### Backward Compatibility for External Metrics + +For external metrics, the adapter also supports the legacy label `resource.labels.project_id` for backward compatibility. However, it is recommended to use the new label `resource.labels.metrics_host_project_id` for clarity and consistency across all metric types. + +If both `resource.labels.metrics_host_project_id` and `resource.labels.project_id` are present for an External metric, the value from `resource.labels.metrics_host_project_id` will be used. + +## Permissions + +Ensure that the service account used by the Custom Metrics Stackdriver Adapter has the necessary permissions (e.g., `monitoring.viewer`) to read metrics from the target project (`my-metrics-project` in the examples above). diff --git a/custom-metrics-stackdriver-adapter/pkg/adapter/provider/provider.go b/custom-metrics-stackdriver-adapter/pkg/adapter/provider/provider.go index 1e8230104..4a52c4364 100644 --- a/custom-metrics-stackdriver-adapter/pkg/adapter/provider/provider.go +++ b/custom-metrics-stackdriver-adapter/pkg/adapter/provider/provider.go @@ -113,13 +113,18 @@ func (p *StackdriverProvider) getRootScopedMetricByName(groupResource schema.Gro if err != nil { return nil, err } + metricsProject, metricSelector, err := p.translator.MetricsHostProjectFromSelector(metricSelector, false) + if err != nil { + return nil, err + } metricName := getCustomMetricName(escapedMetricName) - metricKind, metricValueType, err := p.translator.GetMetricKind(metricName, metricSelector) + metricKind, metricValueType, err := p.translator.GetMetricKind(metricsProject, metricName) if err != nil { return nil, err } stackdriverRequest, err := translator.NewQueryBuilder(p.translator, metricName). WithNodes(&v1.NodeList{Items: []v1.Node{*matchingNode}}). + WithMetricsProject(metricsProject). WithMetricKind(metricKind). WithMetricValueType(metricValueType). WithMetricSelector(metricSelector). @@ -147,8 +152,12 @@ func (p *StackdriverProvider) getRootScopedMetricBySelector(groupResource schema if err != nil { return nil, err } + metricsProject, metricSelector, err := p.translator.MetricsHostProjectFromSelector(metricSelector, false) + if err != nil { + return nil, err + } metricName := getCustomMetricName(escapedMetricName) - metricKind, metricValueType, err := p.translator.GetMetricKind(metricName, metricSelector) + metricKind, metricValueType, err := p.translator.GetMetricKind(metricsProject, metricName) if err != nil { return nil, err } @@ -159,6 +168,7 @@ func (p *StackdriverProvider) getRootScopedMetricBySelector(groupResource schema stackdriverRequest, err := translator.NewQueryBuilder(p.translator, metricName). WithNodes(nodesSlice). WithMetricKind(metricKind). + WithMetricsProject(metricsProject). WithMetricValueType(metricValueType). WithMetricSelector(metricSelector). Build() @@ -189,9 +199,12 @@ func (p *StackdriverProvider) getNamespacedMetricByName(groupResource schema.Gro if err != nil { return nil, err } - + metricsProject, metricSelector, err := p.translator.MetricsHostProjectFromSelector(metricSelector, false) + if err != nil { + return nil, err + } metricName := getCustomMetricName(escapedMetricName) - metricKind, metricValueType, err := p.translator.GetMetricKind(metricName, metricSelector) + metricKind, metricValueType, err := p.translator.GetMetricKind(metricsProject, metricName) if err != nil { return nil, err } @@ -201,6 +214,7 @@ func (p *StackdriverProvider) getNamespacedMetricByName(groupResource schema.Gro stackdriverRequest, err := queryBuilder. WithPods(pods). WithMetricKind(metricKind). + WithMetricsProject(metricsProject). WithMetricValueType(metricValueType). WithMetricSelector(metricSelector). WithNamespace(namespace). @@ -219,6 +233,7 @@ func (p *StackdriverProvider) getNamespacedMetricByName(groupResource schema.Gro AsContainerType(). WithPods(pods). WithMetricKind(metricKind). + WithMetricsProject(metricsProject). WithMetricValueType(metricValueType). WithMetricSelector(metricSelector). WithNamespace(namespace). @@ -249,10 +264,14 @@ func (p *StackdriverProvider) getNamespacedMetricBySelector(groupResource schema if err != nil { return nil, err } + metricsProject, metricSelector, err := p.translator.MetricsHostProjectFromSelector(metricSelector, false) + if err != nil { + return nil, err + } metricName := getCustomMetricName(escapedMetricName) klog.V(4).Infof("Querying for metric: %s", metricName) - metricKind, metricValueType, err := p.translator.GetMetricKind(metricName, metricSelector) + metricKind, metricValueType, err := p.translator.GetMetricKind(metricsProject, metricName) if err != nil { return nil, err } @@ -260,6 +279,7 @@ func (p *StackdriverProvider) getNamespacedMetricBySelector(groupResource schema queryBuilder := translator.NewQueryBuilder(p.translator, metricName). WithMetricKind(metricKind). WithMetricSelector(metricSelector). + WithMetricsProject(metricsProject). WithMetricValueType(metricValueType). WithNamespace(namespace) @@ -290,6 +310,7 @@ func (p *StackdriverProvider) getNamespacedMetricBySelector(groupResource schema AsContainerType(). WithPods(podsSlice). WithMetricKind(metricKind). + WithMetricsProject(metricsProject). WithMetricValueType(metricValueType). WithMetricSelector(metricSelector). WithNamespace(namespace). @@ -335,17 +356,22 @@ func (p *StackdriverProvider) GetExternalMetric(ctx context.Context, namespace s } } + metricsProject, metricSelector, err := p.translator.MetricsHostProjectFromSelector(metricSelector, true) + if err != nil { + return nil, err + } + // Proceed to do a fresh fetch for metrics since one of the following is true at this point // a) externalMetricCache is disabled // b) the key was never added to the cache // c) the key was in the cache, but its corrupt or its TTL has expired metricNameEscaped := info.Metric metricName := getExternalMetricName(metricNameEscaped) - metricKind, metricValueType, err := p.translator.GetMetricKind(metricName, metricSelector) + metricKind, metricValueType, err := p.translator.GetMetricKind(metricsProject, metricName) if err != nil { return nil, err } - stackdriverRequest, err := p.translator.GetExternalMetricRequest(metricName, metricKind, metricValueType, metricSelector) + stackdriverRequest, err := p.translator.GetExternalMetricRequest(metricsProject, metricName, metricKind, metricValueType, metricSelector) if err != nil { return nil, err } diff --git a/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder.go b/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder.go index a44e056e7..65e3c6d9a 100644 --- a/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder.go +++ b/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder.go @@ -171,6 +171,7 @@ func (nc nodeValues) getNodeNames() []string { // use NewQueryBuilder() to initialize type QueryBuilder struct { translator *Translator // translator provides configurations to filter + metricProject string // The project where the metrics are stored. metricName string // metricName is the metric name to filter metricKind string // metricKind is the metric kind to filter metricValueType string // metricValueType is the metric value type to filter @@ -192,11 +193,17 @@ type QueryBuilder struct { // queryBuilder := NewQueryBuilder(NewTranslator(...), "custom.googleapis.com/foo") func NewQueryBuilder(translator *Translator, metricName string) QueryBuilder { return QueryBuilder{ - translator: translator, - metricName: metricName, + translator: translator, + metricName: metricName, + metricProject: translator.config.Project, } } +func (qb QueryBuilder) WithMetricsProject(projectID string) QueryBuilder { + qb.metricProject = projectID + return qb +} + // WithMetricKind adds a metric kind filter to the QueryBuilder // // Example: @@ -344,10 +351,10 @@ func (qb QueryBuilder) validate() error { return apierr.NewInternalError(fmt.Errorf("invalid nodes parameter is set to QueryBuilder")) } if qb.namespace != "" { - return apierr.NewInternalError(fmt.Errorf("both nodes and namespace are provided, expect only one of them.")) + return apierr.NewInternalError(fmt.Errorf("both nodes and namespace are provided, expect only one of them")) } if !qb.pods.isPodValuesEmpty() { - return apierr.NewInternalError(fmt.Errorf("both nodes and pods are provided, expect only one of them.")) + return apierr.NewInternalError(fmt.Errorf("both nodes and pods are provided, expect only one of them")) } } else { // pod metric @@ -453,14 +460,14 @@ func (qb QueryBuilder) Build() (*stackdriver.ProjectsTimeSeriesListCall, error) filter := qb.composeFilter() if qb.metricSelector.Empty() { - return qb.translator.createListTimeseriesRequest(filter, qb.metricKind, qb.metricValueType, ""), nil + return qb.translator.createListTimeseriesRequest(filter, qb.metricKind, qb.metricProject, qb.metricValueType, ""), nil } filterForSelector, reducer, err := qb.translator.filterForSelector(qb.metricSelector, allowedCustomMetricsLabelPrefixes, allowedCustomMetricsFullLabelNames) if err != nil { return nil, err } - return qb.translator.createListTimeseriesRequest(joinFilters(filterForSelector, filter), qb.metricKind, qb.metricValueType, reducer), nil + return qb.translator.createListTimeseriesRequest(joinFilters(filterForSelector, filter), qb.metricKind, qb.metricProject, qb.metricValueType, reducer), nil } // NewTranslator creates a Translator @@ -484,23 +491,21 @@ func NewTranslator(service *stackdriver.Service, gceConf *config.GceConfig, rate } // GetExternalMetricRequest returns Stackdriver request for query for external metric. -func (t *Translator) GetExternalMetricRequest(metricName, metricKind, metricValueType string, metricSelector labels.Selector) (*stackdriver.ProjectsTimeSeriesListCall, error) { +func (t *Translator) GetExternalMetricRequest(metricProject, metricName, metricKind, metricValueType string, metricSelector labels.Selector) (*stackdriver.ProjectsTimeSeriesListCall, error) { if metricValueType == "DISTRIBUTION" && !t.supportDistributions { return nil, apierr.NewBadRequest("Distributions are not supported") } - metricProject, err := t.GetExternalMetricProject(metricSelector) - if err != nil { - return nil, err - } - filterForMetric := t.filterForMetric(metricName) - if metricSelector.Empty() { - return t.createListTimeseriesRequest(filterForMetric, metricKind, metricValueType, ""), nil - } - filterForSelector, reducer, err := t.filterForSelector(metricSelector, allowedExternalMetricsLabelPrefixes, allowedExternalMetricsFullLabelNames) - if err != nil { - return nil, err + filter := t.filterForMetric(metricName) + reducer := "" + if !metricSelector.Empty() { + filterForSelector, reducerForSelector, err := t.filterForSelector(metricSelector, allowedExternalMetricsLabelPrefixes, allowedExternalMetricsFullLabelNames) + if err != nil { + return nil, err + } + filter = joinFilters(filter, filterForSelector) + reducer = reducerForSelector } - return t.createListTimeseriesRequestProject(joinFilters(filterForMetric, filterForSelector), metricKind, metricProject, metricValueType, reducer), nil + return t.createListTimeseriesRequest(filter, metricKind, metricProject, metricValueType, reducer), nil } // ListMetricDescriptors returns Stackdriver request for all custom metrics descriptors. @@ -517,30 +522,15 @@ func (t *Translator) ListMetricDescriptors(fallbackForContainerMetrics bool) *st } // GetMetricKind returns metricKind for metric metricName, obtained from Stackdriver Monitoring API. -func (t *Translator) GetMetricKind(metricName string, metricSelector labels.Selector) (string, string, error) { - metricProj := t.config.Project +func (t *Translator) GetMetricKind(metricsProject string, metricName string) (string, string, error) { cacheKey := metricKindCacheKey{ - project: metricProj, + project: metricsProject, name: metricName, } if value, ok := t.metricKindCache.get(cacheKey); ok { return value.MetricKind, value.ValueType, nil } - - requirements, selectable := metricSelector.Requirements() - if !selectable { - return "", "", apierr.NewBadRequest(fmt.Sprintf("Label selector is impossible to match: %s", metricSelector)) - } - for _, req := range requirements { - if req.Key() == "resource.labels.project_id" { - if req.Operator() == selection.Equals || req.Operator() == selection.DoubleEquals { - metricProj = req.Values().List()[0] - break - } - return "", "", NewLabelNotAllowedError(fmt.Sprintf("Project selector must use '=' or '==': You used %s", req.Operator())) - } - } - response, err := t.service.Projects.MetricDescriptors.Get(fmt.Sprintf("projects/%s/metricDescriptors/%s", metricProj, metricName)).Do() + response, err := t.service.Projects.MetricDescriptors.Get(fmt.Sprintf("projects/%s/metricDescriptors/%s", metricsProject, metricName)).Do() if err != nil { return "", "", NewNoSuchMetricError(metricName, err) } @@ -551,18 +541,34 @@ func (t *Translator) GetMetricKind(metricName string, metricSelector labels.Sele return response.MetricKind, response.ValueType, nil } -// GetExternalMetricProject If the metric has "resource.labels.project_id" as a selector, then use a different project -func (t *Translator) GetExternalMetricProject(metricSelector labels.Selector) (string, error) { - requirements, _ := metricSelector.Requirements() +// MetricsHostProjectFromSelector returns the project id from the metric selector, if it's specified. +// It supports both "resource.labels.metrics_host_project_id" and the legacy "resource.labels.project_id". +// If "resource.labels.metrics_host_project_id" is found, it's removed from the selector. +func (t *Translator) MetricsHostProjectFromSelector(metricSelector labels.Selector, lookupLegacySelector bool) (string, labels.Selector, error) { + const hostMetricProjectLabel = "resource.labels.metrics_host_project_id" + requirements, selectable := metricSelector.Requirements() + if !selectable { + return "", metricSelector, apierr.NewBadRequest(fmt.Sprintf("Label selector is impossible to match: %s", metricSelector)) + } for _, req := range requirements { - if req.Key() == "resource.labels.project_id" { + if req.Key() == hostMetricProjectLabel { if req.Operator() == selection.Equals || req.Operator() == selection.DoubleEquals { - return req.Values().List()[0], nil + return req.Values().List()[0], removeKeyFromSelector(metricSelector, hostMetricProjectLabel), nil } - return "", NewLabelNotAllowedError(fmt.Sprintf("Project selector must use '=' or '==': You used %s", req.Operator())) + return "", metricSelector, NewLabelNotAllowedError(fmt.Sprintf("Metrics project selector must use '=' or '==': You used %s", req.Operator())) } } - return t.config.Project, nil + if lookupLegacySelector { + for _, req := range requirements { + if req.Key() == "resource.labels.project_id" { + if req.Operator() == selection.Equals || req.Operator() == selection.DoubleEquals { + return req.Values().List()[0], metricSelector, nil + } + return "", metricSelector, NewLabelNotAllowedError(fmt.Sprintf("Metrics project selector must use '=' or '==': You used %s", req.Operator())) + } + } + } + return t.config.Project, metricSelector, nil } func isAllowedLabelName(labelName string, allowedLabelPrefixes []string, allowedFullLabelNames []string) bool { @@ -579,6 +585,31 @@ func isAllowedLabelName(labelName string, allowedLabelPrefixes []string, allowed return false } +// removeKeyFromSelector returns a new selector that doesn't contain any requirement with the given key. +func removeKeyFromSelector(selector labels.Selector, key string) labels.Selector { + if selector == nil || selector.Empty() { + return selector + } + + reqs, selectable := selector.Requirements() + if !selectable { + return selector + } + + var newReqs []labels.Requirement + for _, r := range reqs { + if r.Key() != key { + newReqs = append(newReqs, r) + } + } + + if len(newReqs) == len(reqs) { + return selector + } + + return labels.NewSelector().Add(newReqs...) +} + func splitMetricLabel(labelName string, allowedLabelPrefixes []string) (string, string, error) { for _, prefix := range allowedLabelPrefixes { if strings.HasPrefix(labelName, prefix+".") { @@ -620,21 +651,6 @@ func (t *Translator) filterForMetric(metricName string) string { return fmt.Sprintf("metric.type = %q", metricName) } -// Deprecated, use FilterBuilder instead -func (t *Translator) filterForAnyPod() string { - return "resource.type = \"k8s_pod\"" -} - -// Deprecated, use FilterBuilder instead -func (t *Translator) filterForAnyNode() string { - return "resource.type = \"k8s_node\"" -} - -// Deprecated, use FilterBuilder instead -func (t *Translator) filterForAnyContainer() string { - return "resource.type = \"k8s_container\"" -} - // Deprecated, use FilterBuilder instead func (t *Translator) filterForAnyResource(fallbackForContainerMetrics bool) string { if fallbackForContainerMetrics { @@ -643,23 +659,6 @@ func (t *Translator) filterForAnyResource(fallbackForContainerMetrics bool) stri return "resource.type = one_of(\"k8s_pod\",\"k8s_node\")" } -// Deprecated, use FilterBuilder instead -// The namespace string can be empty. If so, all namespaces are allowed. -func (t *Translator) filterForPods(podNames []string, namespace string) string { - if len(podNames) == 0 { - klog.Fatalf("createFilterForPods called with empty list of pod names") - } else if len(podNames) == 1 { - if namespace == AllNamespaces { - return fmt.Sprintf("resource.labels.pod_name = %s", podNames[0]) - } - return fmt.Sprintf("resource.labels.namespace_name = %q AND resource.labels.pod_name = %s", namespace, podNames[0]) - } - if namespace == AllNamespaces { - return fmt.Sprintf("resource.labels.pod_name = one_of(%s)", strings.Join(podNames, ",")) - } - return fmt.Sprintf("resource.labels.namespace_name = %q AND resource.labels.pod_name = one_of(%s)", namespace, strings.Join(podNames, ",")) -} - // Deprecated, use FilterBuilder instead func (t *Translator) filterForNodes(nodeNames []string) string { if len(nodeNames) == 0 { @@ -684,16 +683,6 @@ func (t *Translator) legacyFilterForAnyPod() string { return "resource.labels.pod_id != \"\" AND resource.labels.pod_id != \"machine\"" } -// Deprecated, use FilterBuilder instead -func (t *Translator) legacyFilterForPods(podIDs []string) string { - if len(podIDs) == 0 { - klog.Fatalf("createFilterForIDs called with empty list of pod IDs") - } else if len(podIDs) == 1 { - return fmt.Sprintf("resource.labels.pod_id = %s", podIDs[0]) - } - return fmt.Sprintf("resource.labels.pod_id = one_of(%s)", strings.Join(podIDs, ",")) -} - func (t *Translator) filterForSelector(metricSelector labels.Selector, allowedLabelPrefixes []string, allowedFullLabelNames []string) (string, string, error) { requirements, selectable := metricSelector.Requirements() if !selectable { @@ -767,7 +756,7 @@ func (t *Translator) filterForSelector(metricSelector labels.Selector, allowedLa if isAllowedLabelName(req.Key(), allowedLabelPrefixes, allowedFullLabelNames) { value, err := strconv.ParseInt(l[0], 10, 64) if err != nil { - return "", "", apierr.NewInternalError(fmt.Errorf("Unexpected error: value %s could not be parsed to integer", l[0])) + return "", "", apierr.NewInternalError(fmt.Errorf("unexpected error: value %s could not be parsed to integer", l[0])) } filters = append(filters, fmt.Sprintf("%s > %v", req.Key(), value)) } else { @@ -777,7 +766,7 @@ func (t *Translator) filterForSelector(metricSelector labels.Selector, allowedLa if isAllowedLabelName(req.Key(), allowedLabelPrefixes, allowedFullLabelNames) { value, err := strconv.ParseInt(l[0], 10, 64) if err != nil { - return "", "", apierr.NewInternalError(fmt.Errorf("Unexpected error: value %s could not be parsed to integer", l[0])) + return "", "", apierr.NewInternalError(fmt.Errorf("unexpected error: value %s could not be parsed to integer", l[0])) } filters = append(filters, fmt.Sprintf("%s < %v", req.Key(), value)) } else { @@ -802,12 +791,7 @@ func (t *Translator) getMetricLabels(series *stackdriver.TimeSeries) map[string] return metricLabels } -func (t *Translator) createListTimeseriesRequest(filter, metricKind, metricValueType, reducer string) *stackdriver.ProjectsTimeSeriesListCall { - return t.createListTimeseriesRequestProject(filter, metricKind, t.config.Project, metricValueType, reducer) -} - -func (t *Translator) createListTimeseriesRequestProject(filter, metricKind, metricProject, metricValueType, reducer string) *stackdriver.ProjectsTimeSeriesListCall { - project := fmt.Sprintf("projects/%s", metricProject) +func (t *Translator) createListTimeseriesRequest(filter, metricKind, metricProject, metricValueType, reducer string) *stackdriver.ProjectsTimeSeriesListCall { endTime := t.clock.Now() startTime := endTime.Add(-t.reqWindow) // use "ALIGN_NEXT_OLDER" by default, i.e. for metricKind "GAUGE" @@ -820,7 +804,9 @@ func (t *Translator) createListTimeseriesRequestProject(filter, metricKind, metr if metricValueType == "DISTRIBUTION" { aligner = "ALIGN_DELTA" } - ptslc := t.service.Projects.TimeSeries.List(project).Filter(filter). + ptslc := t.service.Projects.TimeSeries. + List(fmt.Sprintf("projects/%s", metricProject)). + Filter(filter). IntervalStartTime(startTime.Format(time.RFC3339)). IntervalEndTime(endTime.Format(time.RFC3339)). AggregationPerSeriesAligner(aligner). diff --git a/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder_test.go b/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder_test.go index 03286ecf9..f9c7577d2 100644 --- a/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder_test.go +++ b/custom-metrics-stackdriver-adapter/pkg/adapter/translator/query_builder_test.go @@ -34,13 +34,6 @@ import ( "k8s.io/apimachinery/pkg/selection" ) -func TestQueryBuilder_nil_translator(t *testing.T) { - _, err := NewQueryBuilder(nil, "my-metric-name").Build() - if err == nil { - t.Error("Expected nil translation error, but found nil") - } -} - func TestQueryBuilder_Both_Pods_PodNames_Provided(t *testing.T) { translator, _ := NewFakeTranslator(2*time.Minute, time.Minute, "my-project", "my-cluster", "my-zone", time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC), true) @@ -70,6 +63,7 @@ func TestTranslator_QueryBuilder_pod_Single(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -111,6 +105,7 @@ func TestTranslator_QueryBuilder_prometheus_Single(t *testing.T) { } metricName := "prometheus.googleapis.com/foo" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -153,6 +148,7 @@ func TestTranslator_QueryBuilder_pod_SingleWithMetricSelector(t *testing.T) { metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -196,6 +192,7 @@ func TestTranslator_QueryBuilder_prometheus_SingleWithMetricSelector(t *testing. metricName := "prometheus.googleapis.com/foo/gauge" metricSelector, _ := labels.Parse("metric.labels.custom=test") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -290,6 +287,7 @@ func TestTranslator_QueryBuilder_pod_Multiple(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -337,6 +335,7 @@ func TestTranslator_QueryBuilder_prometheus_Multiple(t *testing.T) { } metricName := "prometheus.googleapis.com/foo/gauge" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -385,6 +384,7 @@ func TestTranslator_QueryBuilder_pod_MultipleWithMetricSelctor(t *testing.T) { metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -427,6 +427,7 @@ func TestTranslator_QueryBuilder_Container_Single(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). @@ -469,6 +470,7 @@ func TestTranslator_QueryBuilder_Container_SingleWithEmptyNamespace(t *testing.T } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). @@ -534,6 +536,7 @@ func TestTranslator_QueryBuilder_Container_SingleWithMetricSelector(t *testing.T metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("GAUGE"). @@ -600,6 +603,7 @@ func TestTranslator_QueryBuilder_Container_Multiple(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). @@ -648,6 +652,7 @@ func TestTranslator_QueryBuilder_Container_MultipleEmptyNamespace(t *testing.T) } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). @@ -696,6 +701,7 @@ func TestTranslator_QueryBuilder_Container_MultipleWithMetricSelctor(t *testing. metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). @@ -739,6 +745,7 @@ func TestQueryBuilder_Node(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithNodes(&v1.NodeList{Items: []v1.Node{node}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -778,6 +785,7 @@ func TestQueryBuilder_Multiple_Nodes(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithNodes(&v1.NodeList{Items: []v1.Node{node, node}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -817,7 +825,8 @@ func TestQueryBuilder_Node_withMetricSelector(t *testing.T) { } metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test") - request, err := NewQueryBuilder(translator, metricName).WithNodes(&v1.NodeList{Items: []v1.Node{node}}).WithMetricKind("GAUGE").WithMetricValueType("INT64").WithMetricSelector(metricSelector).Build() + request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project").WithNodes(&v1.NodeList{Items: []v1.Node{node}}).WithMetricKind("GAUGE").WithMetricValueType("INT64").WithMetricSelector(metricSelector).Build() if err != nil { t.Errorf("Translation error: %s", err) } @@ -857,6 +866,7 @@ func TestTranslator_QueryBuilder_pod_legacyResourceModel(t *testing.T) { } metricName := "my/custom/metric" request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod1, pod2}}). WithMetricKind("GAUGE"). WithMetricValueType("INT64"). @@ -928,7 +938,7 @@ func TestTranslator_ListMetricDescriptors_legacyResourceType(t *testing.T) { } func TestTranslator_GetExternalMetricRequest_NoSelector(t *testing.T) { translator, sdService := newFakeTranslatorForExternalMetrics(2*time.Minute, time.Minute, "my-project", time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC)) - request, err := translator.GetExternalMetricRequest("custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.NewSelector()) + request, err := translator.GetExternalMetricRequest("my-project", "custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.NewSelector()) if err != nil { t.Fatalf("Translation error: %s", err) } @@ -950,7 +960,7 @@ func TestTranslator_GetExternalMetricRequest_CorrectSelector_Cumulative(t *testi req3, _ := labels.NewRequirement("resource.labels.pod_name", selection.Exists, []string{}) req4, _ := labels.NewRequirement("resource.labels.namespace_name", selection.NotIn, []string{"default", "kube-system"}) req5, _ := labels.NewRequirement("metric.labels.my_label", selection.GreaterThan, []string{"86"}) - request, err := translator.GetExternalMetricRequest("custom.googleapis.com/my/metric/name", "CUMULATIVE", "INT64", labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5)) + request, err := translator.GetExternalMetricRequest("my-project", "custom.googleapis.com/my/metric/name", "CUMULATIVE", "INT64", labels.NewSelector().Add(*req1, *req2, *req3, *req4, *req5)) if err != nil { t.Fatalf("Translation error: %s", err) } @@ -974,7 +984,7 @@ func TestTranslator_GetExternalMetricRequest_DifferentProject(t *testing.T) { translator, sdService := newFakeTranslatorForExternalMetrics(2*time.Minute, time.Minute, "my-project", time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC)) req1, _ := labels.NewRequirement("resource.type", selection.Equals, []string{"k8s_pod"}) req2, _ := labels.NewRequirement("resource.labels.project_id", selection.Equals, []string{"other-project"}) - request, err := translator.GetExternalMetricRequest("custom.googleapis.com/my/metric/name", "CUMULATIVE", "INT64", labels.NewSelector().Add(*req1, *req2)) + request, err := translator.GetExternalMetricRequest("other-project", "custom.googleapis.com/my/metric/name", "CUMULATIVE", "INT64", labels.NewSelector().Add(*req1, *req2)) if err != nil { t.Fatalf("Translation error: %s", err) } @@ -993,7 +1003,7 @@ func TestTranslator_GetExternalMetricRequest_DifferentProject(t *testing.T) { func TestTranslator_GetExternalMetricRequest_InvalidLabel(t *testing.T) { translator, _ := newFakeTranslatorForExternalMetrics(2*time.Minute, time.Minute, "my-project", time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC)) - _, err := translator.GetExternalMetricRequest("custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.SelectorFromSet(labels.Set{ + _, err := translator.GetExternalMetricRequest("my-project", "custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.SelectorFromSet(labels.Set{ "arbitrary-label": "foo", })) expectedError := NewLabelNotAllowedError("arbitrary-label") @@ -1008,7 +1018,7 @@ func TestTranslator_GetExternalMetricRequest_OneInvalidRequirement(t *testing.T) req2, _ := labels.NewRequirement("resource.labels.pod_name", selection.Exists, []string{}) req3, _ := labels.NewRequirement("resource.labels.namespace_name", selection.NotIn, []string{"default", "kube-system"}) req4, _ := labels.NewRequirement("metric.labels.my_label", selection.DoesNotExist, []string{}) - _, err := translator.GetExternalMetricRequest("custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.NewSelector().Add(*req1, *req2, *req3, *req4)) + _, err := translator.GetExternalMetricRequest("my-project", "custom.googleapis.com/my/metric/name", "GAUGE", "INT64", labels.NewSelector().Add(*req1, *req2, *req3, *req4)) expectedError := errors.NewBadRequest("Label selector with operator DoesNotExist is not allowed") if *err.(*errors.StatusError) != *expectedError { t.Errorf("Expected status error: %s, but received: %s", expectedError, err) @@ -1027,6 +1037,7 @@ func TestTranslator_QueryBuilder_pod_Single_Distribution(t *testing.T) { metricName := "my/custom/metric" selector, _ := labels.Parse("reducer=REDUCE_PERCENTILE_50") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("DELTA"). WithMetricValueType("DISTRIBUTION"). @@ -1070,6 +1081,7 @@ func TestTranslator_QueryBuilder_promethus_Single_Distribution(t *testing.T) { metricName := "prometheus.googleapis.com/foo/gauge" selector, _ := labels.Parse("reducer=REDUCE_PERCENTILE_50") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("DELTA"). WithMetricValueType("DISTRIBUTION"). @@ -1257,6 +1269,7 @@ func TestTranslator_QueryBuilder_pod_SingleWithMetricSelector_Distribution(t *te metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test,reducer=REDUCE_PERCENTILE_99") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("DELTA"). WithMetricValueType("DISTRIBUTION"). @@ -1301,6 +1314,7 @@ func TestTranslator_QueryBuilder_Container_Single_Distribution(t *testing.T) { metricName := "my/custom/metric" selector, _ := labels.Parse("reducer=REDUCE_PERCENTILE_50") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("DELTA"). @@ -1345,6 +1359,7 @@ func TestTranslator_GetSDReqForContainer_SingleWithMetricSelector_Distribution(t metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test,reducer=REDUCE_PERCENTILE_99") request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). AsContainerType(). WithPods(&v1.PodList{Items: []v1.Pod{pod}}). WithMetricKind("DELTA"). @@ -1392,7 +1407,7 @@ func TestTranslator_GetMetricKind_KindFoundInCache(t *testing.T) { translator := newFakeTranslatorForGetMetricKind(2*time.Minute, time.Minute, testProjectID, time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC), sdService, cache) - kind, value, err := translator.GetMetricKind("pubsub.googleapis.com/subscription/num_undelivered_messages", labels.Everything()) + kind, value, err := translator.GetMetricKind(testProjectID, "pubsub.googleapis.com/subscription/num_undelivered_messages") if err != nil { t.Errorf("Unexpected error: %s", err) } @@ -1419,7 +1434,7 @@ func TestTranslator_GetMetricKind_KindNotFoundInCache(t *testing.T) { cache := newMetricKindCache(2, 10*time.Minute) translator := newFakeTranslatorForGetMetricKind(2*time.Minute, time.Minute, testProjectID, time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC), sdService, cache) - kind, value, err := translator.GetMetricKind(metricKind, labels.Everything()) + kind, value, err := translator.GetMetricKind(testProjectID, metricKind) if err != nil { t.Errorf("Unexpected error: %s", err) } @@ -1481,7 +1496,13 @@ func TestQueryBuilder_Node_Single_Distribution(t *testing.T) { } metricName := "my/custom/metric" metricSelector, _ := labels.Parse("reducer=REDUCE_PERCENTILE_95") - request, err := NewQueryBuilder(translator, metricName).WithNodes(&v1.NodeList{Items: []v1.Node{node}}).WithMetricKind("DELTA").WithMetricValueType("DISTRIBUTION").WithMetricSelector(metricSelector).Build() + request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project"). + WithNodes(&v1.NodeList{Items: []v1.Node{node}}). + WithMetricsProject("my-project"). + WithMetricKind("DELTA"). + WithMetricValueType("DISTRIBUTION"). + WithMetricSelector(metricSelector).Build() if err != nil { t.Errorf("Translation error: %s", err) } @@ -1517,7 +1538,8 @@ func TestQueryBuilder_Node_SingleWithMetricSelector_Distribution(t *testing.T) { } metricName := "my/custom/metric" metricSelector, _ := labels.Parse("metric.labels.custom=test,reducer=REDUCE_PERCENTILE_95") - request, err := NewQueryBuilder(translator, metricName).WithNodes(&v1.NodeList{Items: []v1.Node{node}}).WithMetricKind("DELTA").WithMetricValueType("DISTRIBUTION").WithMetricSelector(metricSelector).Build() + request, err := NewQueryBuilder(translator, metricName). + WithMetricsProject("my-project").WithNodes(&v1.NodeList{Items: []v1.Node{node}}).WithMetricKind("DELTA").WithMetricValueType("DISTRIBUTION").WithMetricSelector(metricSelector).Build() if err != nil { t.Errorf("Translation error: %s", err) } @@ -1542,3 +1564,152 @@ func TestQueryBuilder_Node_SingleWithMetricSelector_Distribution(t *testing.T) { t.Errorf("Unexpected result. Expected: \n%v,\n received: \n%v", expectedRequest, request) } } + +func TestRemoveKeyFromSelector(t *testing.T) { + testCases := []struct { + name string + selector string + keyToRemove string + expectedResult string + }{ + { + name: "remove from middle", + selector: "key1=val1,key2=val2,key3=val3", + keyToRemove: "key2", + expectedResult: "key1=val1,key3=val3", + }, + { + name: "remove non-existent key", + selector: "key1=val1,key2=val2", + keyToRemove: "key3", + expectedResult: "key1=val1,key2=val2", + }, + { + name: "remove from empty selector", + selector: "", + keyToRemove: "key1", + expectedResult: "", + }, + { + name: "remove last key", + selector: "key1=val1", + keyToRemove: "key1", + expectedResult: "", + }, + { + name: "remove from selector with different operators", + selector: "key1=val1,key2 in (v1,v2),!key3", + keyToRemove: "key2", + expectedResult: "key1=val1,!key3", + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + selector, err := labels.Parse(tc.selector) + if err != nil { + t.Fatalf("failed to parse selector %q: %v", tc.selector, err) + } + + newSelector := removeKeyFromSelector(selector, tc.keyToRemove) + + if newSelector.String() != tc.expectedResult { + t.Errorf("RemoveKeyFromSelector(%q, %q) = %q, want %q", tc.selector, tc.keyToRemove, newSelector.String(), tc.expectedResult) + } + }) + } +} + +func TestTranslator_MetricsHostProjectFromSelector(t *testing.T) { + translator, _ := newFakeTranslatorForExternalMetrics(2*time.Minute, time.Minute, "my-project", time.Date(2017, 1, 2, 13, 2, 0, 0, time.UTC)) + testCases := []struct { + name string + selector string + lookupLegacy bool + expectedProject string + expectedSelector string + expectError bool + }{ + { + name: "Default project", + selector: "metric.labels.foo=bar", + lookupLegacy: true, + expectedProject: "my-project", + expectedSelector: "metric.labels.foo=bar", + }, + { + name: "New project label", + selector: "resource.labels.metrics_host_project_id=other-project,metric.labels.foo=bar", + lookupLegacy: true, + expectedProject: "other-project", + expectedSelector: "metric.labels.foo=bar", + }, + { + name: "Legacy project label - lookup enabled", + selector: "resource.labels.project_id=other-project,metric.labels.foo=bar", + lookupLegacy: true, + expectedProject: "other-project", + expectedSelector: "resource.labels.project_id=other-project,metric.labels.foo=bar", + }, + { + name: "Legacy project label - lookup disabled", + selector: "resource.labels.project_id=other-project,metric.labels.foo=bar", + lookupLegacy: false, + expectedProject: "my-project", + expectedSelector: "resource.labels.project_id=other-project,metric.labels.foo=bar", + }, + { + name: "New label takes precedence", + selector: "resource.labels.metrics_host_project_id=new-project,resource.labels.project_id=legacy-project", + lookupLegacy: true, + expectedProject: "new-project", + expectedSelector: "resource.labels.project_id=legacy-project", + }, + { + name: "Invalid operator for new label", + selector: "resource.labels.metrics_host_project_id!=other-project", + lookupLegacy: true, + expectError: true, + }, + { + name: "Invalid operator for legacy label", + selector: "resource.labels.project_id!=other-project", + lookupLegacy: true, + expectError: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + selector, err := labels.Parse(tc.selector) + if err != nil { + t.Fatalf("failed to parse selector %q: %v", tc.selector, err) + } + + project, newSelector, err := translator.MetricsHostProjectFromSelector(selector, tc.lookupLegacy) + + if tc.expectError { + if err == nil { + t.Errorf("Expected an error but got none") + } + return + } + + if err != nil { + t.Fatalf("Unexpected error: %v", err) + } + + if project != tc.expectedProject { + t.Errorf("Expected project %q, but got %q", tc.expectedProject, project) + } + + expectedSelector, err := labels.Parse(tc.expectedSelector) + if err != nil { + t.Fatalf("failed to parse expected selector %q: %v", tc.expectedSelector, err) + } + if !reflect.DeepEqual(newSelector, expectedSelector) { + t.Errorf("Expected selector %q, but got %q", expectedSelector, newSelector) + } + }) + } +}