From dcbeb30163a4beca08c1663693939988c250a8a8 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 30 Jan 2026 18:28:49 +0900 Subject: [PATCH 1/5] return 415 when prw2 is disabled Signed-off-by: SungJin1212 --- CHANGELOG.md | 1 + pkg/util/push/push.go | 64 ++++++++++++++++----------------- pkg/util/push/push_test.go | 73 ++++++++++++++++++++++++-------------- 3 files changed, 80 insertions(+), 58 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f2b02e6a64..53dba67a3ad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ * [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` that enables adding `__unit__` and `__type__` labels for remote write v2 and OTLP requests. This is a breaking change; the `-distributor.otlp.enable-type-and-unit-labels` flag is now deprecated, operates as a no-op, and has been consolidated into this new flag. #7077 * [FEATURE] Querier: Add experimental projection pushdown support in Parquet Queryable. #7152 * [FEATURE] Ingester: Add experimental active series queried metric. #7173 +* [ENHANCEMENT] Distributor: If remote write v2 is disabled, explicitly return HTTP 415 (Unsupported Media Type) for Remote Write V2 requests instead of attempting to parse them as V1. #7238 * [ENHANCEMENT] Querier: Add `-querier.store-gateway-series-batch-size` flag to configure the maximum number of series to be batched in a single gRPC response message from Store Gateways. #7203 * [ENHANCEMENT] HATracker: Add `-distributor.ha-tracker.enable-startup-sync` flag. If enabled, the ha-tracker fetches all tracked keys on startup to populate the local cache. #7213 * [ENHANCEMENT] Ingester: Add support for ingesting Native Histogram with Custom Buckets. #7191 diff --git a/pkg/util/push/push.go b/pkg/util/push/push.go index d15c5e51e2b..05e59f4b1c0 100644 --- a/pkg/util/push/push.go +++ b/pkg/util/push/push.go @@ -137,43 +137,43 @@ func Handler(remoteWrite2Enabled bool, maxRecvMsgSize int, overrides *validation } } - if remoteWrite2Enabled { - // follow Prometheus https://github.com/prometheus/prometheus/blob/v3.3.1/storage/remote/write_handler.go#L121 - contentType := r.Header.Get("Content-Type") - if contentType == "" { - contentType = appProtoContentType - } + // follow Prometheus https://github.com/prometheus/prometheus/blob/v3.3.1/storage/remote/write_handler.go#L121 + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = appProtoContentType + } - msgType, err := remote.ParseProtoMsg(contentType) - if err != nil { - level.Error(logger).Log("Error decoding remote write request", "err", err) - http.Error(w, err.Error(), http.StatusUnsupportedMediaType) - return - } + msgType, err := remote.ParseProtoMsg(contentType) + if err != nil { + level.Error(logger).Log("Error decoding remote write request", "err", err) + http.Error(w, err.Error(), http.StatusUnsupportedMediaType) + return + } - if msgType != remote.WriteV1MessageType && msgType != remote.WriteV2MessageType { - level.Error(logger).Log("Not accepted msg type", "msgType", msgType, "err", err) - http.Error(w, err.Error(), http.StatusUnsupportedMediaType) - return - } + if msgType != remote.WriteV1MessageType && msgType != remote.WriteV2MessageType { + level.Error(logger).Log("Not accepted msg type", "msgType", msgType, "err", err) + http.Error(w, err.Error(), http.StatusUnsupportedMediaType) + return + } - enc := r.Header.Get("Content-Encoding") - if enc == "" { - } else if enc != compression.Snappy { - err := fmt.Errorf("%v encoding (compression) is not accepted by this server; only %v is acceptable", enc, compression.Snappy) - level.Error(logger).Log("Error decoding remote write request", "err", err) - http.Error(w, err.Error(), http.StatusUnsupportedMediaType) - return - } + enc := r.Header.Get("Content-Encoding") + if enc == "" { + } else if enc != compression.Snappy { + err := fmt.Errorf("%v encoding (compression) is not accepted by this server; only %v is acceptable", enc, compression.Snappy) + level.Error(logger).Log("Error decoding remote write request", "err", err) + http.Error(w, err.Error(), http.StatusUnsupportedMediaType) + return + } - switch msgType { - case remote.WriteV1MessageType: - handlePRW1() - case remote.WriteV2MessageType: - handlePRW2() - } - } else { + switch msgType { + case remote.WriteV1MessageType: handlePRW1() + case remote.WriteV2MessageType: + if !remoteWrite2Enabled { + http.Error(w, "Remote Write V2 is disabled", http.StatusUnsupportedMediaType) + return + } + handlePRW2() } }) } diff --git a/pkg/util/push/push_test.go b/pkg/util/push/push_test.go index bf21863eea4..008f02de54b 100644 --- a/pkg/util/push/push_test.go +++ b/pkg/util/push/push_test.go @@ -484,13 +484,13 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { overrides := validation.NewOverrides(limits, nil) sourceIPs, _ := middleware.NewSourceIPs("SomeField", "(.*)") - handler := Handler(true, 100000, overrides, sourceIPs, verifyWriteRequestHandler(t, cortexpb.API)) tests := []struct { - description string - reqHeaders map[string]string - expectedCode int - isV2 bool + description string + reqHeaders map[string]string + expectedCode int + isV2 bool + remoteWrite2Enabled bool }{ { description: "[RW 2.0] correct content-type", @@ -499,8 +499,9 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusNoContent, - isV2: true, + expectedCode: http.StatusNoContent, + isV2: true, + remoteWrite2Enabled: true, }, { description: "[RW 1.0] correct content-type", @@ -509,8 +510,8 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "0.1.0", }, - expectedCode: http.StatusOK, - isV2: false, + expectedCode: http.StatusOK, + isV2: false, }, { description: "[RW 2.0] wrong content-type", @@ -519,8 +520,8 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusUnsupportedMediaType, - isV2: true, + expectedCode: http.StatusUnsupportedMediaType, + isV2: true, }, { description: "[RW 2.0] wrong content-type", @@ -529,8 +530,9 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusUnsupportedMediaType, - isV2: true, + expectedCode: http.StatusUnsupportedMediaType, + isV2: true, + remoteWrite2Enabled: true, }, { description: "[RW 2.0] wrong content-encoding", @@ -539,13 +541,26 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "zstd", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusUnsupportedMediaType, - isV2: true, + expectedCode: http.StatusUnsupportedMediaType, + isV2: true, + remoteWrite2Enabled: true, }, { - description: "no header, should treated as RW 1.0", - expectedCode: http.StatusOK, - isV2: false, + description: "[RW 2.0] V2 disabled", + reqHeaders: map[string]string{ + "Content-Type": appProtoV2ContentType, + "Content-Encoding": "snappy", + remoteWriteVersionHeader: "2.0.0", + }, + expectedCode: http.StatusUnsupportedMediaType, + isV2: true, + remoteWrite2Enabled: false, + }, + { + description: "no header, should treated as RW 1.0", + expectedCode: http.StatusOK, + isV2: false, + remoteWrite2Enabled: true, }, { description: "missing content-type, should treated as RW 1.0", @@ -553,8 +568,9 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusOK, - isV2: false, + expectedCode: http.StatusOK, + isV2: false, + remoteWrite2Enabled: true, }, { description: "missing content-encoding", @@ -562,8 +578,9 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Type": appProtoV2ContentType, remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusNoContent, - isV2: true, + expectedCode: http.StatusNoContent, + isV2: true, + remoteWrite2Enabled: true, }, { description: "missing remote write version, should treated based on Content-type", @@ -571,8 +588,9 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Type": appProtoV2ContentType, "Content-Encoding": "snappy", }, - expectedCode: http.StatusNoContent, - isV2: true, + expectedCode: http.StatusNoContent, + isV2: true, + remoteWrite2Enabled: true, }, { description: "missing remote write version, should treated based on Content-type", @@ -580,13 +598,16 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Type": appProtoV1ContentType, "Content-Encoding": "snappy", }, - expectedCode: http.StatusOK, - isV2: false, + expectedCode: http.StatusOK, + isV2: false, + remoteWrite2Enabled: true, }, } for _, test := range tests { t.Run(test.description, func(t *testing.T) { + handler := Handler(test.remoteWrite2Enabled, 100000, overrides, sourceIPs, verifyWriteRequestHandler(t, cortexpb.API)) + if test.isV2 { ctx := context.Background() ctx = user.InjectOrgID(ctx, "user-1") From b3720f0f21acecfc6363b7fa895aeed0792f6d2a Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 30 Jan 2026 18:34:27 +0900 Subject: [PATCH 2/5] go fmt Signed-off-by: SungJin1212 --- pkg/util/push/push_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/util/push/push_test.go b/pkg/util/push/push_test.go index 008f02de54b..cf264705721 100644 --- a/pkg/util/push/push_test.go +++ b/pkg/util/push/push_test.go @@ -510,8 +510,8 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "0.1.0", }, - expectedCode: http.StatusOK, - isV2: false, + expectedCode: http.StatusOK, + isV2: false, }, { description: "[RW 2.0] wrong content-type", @@ -520,8 +520,8 @@ func TestHandler_ContentTypeAndEncoding(t *testing.T) { "Content-Encoding": "snappy", remoteWriteVersionHeader: "2.0.0", }, - expectedCode: http.StatusUnsupportedMediaType, - isV2: true, + expectedCode: http.StatusUnsupportedMediaType, + isV2: true, }, { description: "[RW 2.0] wrong content-type", From 851f172a35652e11a04fb133294c3eb0a060c1b6 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Fri, 30 Jan 2026 18:54:46 +0900 Subject: [PATCH 3/5] fix test Signed-off-by: SungJin1212 --- integration/remote_write_v2_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration/remote_write_v2_test.go b/integration/remote_write_v2_test.go index ed0f1902fc6..7b00e0eb7a2 100644 --- a/integration/remote_write_v2_test.go +++ b/integration/remote_write_v2_test.go @@ -194,7 +194,7 @@ func TestIngest_SenderSendPRW2_DistributorNotAllowPRW2(t *testing.T) { symbols1, series, _ := e2e.GenerateSeriesV2("test_series", now, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "foo", Value: "bar"}) _, err = c.PushV2(symbols1, series) require.Error(t, err) - require.Contains(t, err.Error(), "sent v2 request; got 2xx, but PRW 2.0 response header statistics indicate 0 samples, 0 histograms and 0 exemplars were accepted") + require.Contains(t, err.Error(), "server returned HTTP status 415 Unsupported Media Type: Remote Write V2 is disabled") // sample result, err := c.Query("test_series", now) From cb3eca97a4f8ec58daf7aaa3d5c2a90a0336a0f1 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Sun, 1 Feb 2026 20:21:11 +0900 Subject: [PATCH 4/5] move changelog to bugfix Signed-off-by: SungJin1212 --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 53dba67a3ad..2164d604b8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,7 +13,6 @@ * [FEATURE] Distributor: Add a per-tenant flag `-distributor.enable-type-and-unit-labels` that enables adding `__unit__` and `__type__` labels for remote write v2 and OTLP requests. This is a breaking change; the `-distributor.otlp.enable-type-and-unit-labels` flag is now deprecated, operates as a no-op, and has been consolidated into this new flag. #7077 * [FEATURE] Querier: Add experimental projection pushdown support in Parquet Queryable. #7152 * [FEATURE] Ingester: Add experimental active series queried metric. #7173 -* [ENHANCEMENT] Distributor: If remote write v2 is disabled, explicitly return HTTP 415 (Unsupported Media Type) for Remote Write V2 requests instead of attempting to parse them as V1. #7238 * [ENHANCEMENT] Querier: Add `-querier.store-gateway-series-batch-size` flag to configure the maximum number of series to be batched in a single gRPC response message from Store Gateways. #7203 * [ENHANCEMENT] HATracker: Add `-distributor.ha-tracker.enable-startup-sync` flag. If enabled, the ha-tracker fetches all tracked keys on startup to populate the local cache. #7213 * [ENHANCEMENT] Ingester: Add support for ingesting Native Histogram with Custom Buckets. #7191 @@ -38,6 +37,7 @@ * [ENHANCEMENT] Ingester: Add fetch timeout for Ingester expanded postings cache. #7185 * [ENHANCEMENT] Ingester: Add feature flag to collect metrics of how expensive an unoptimized regex matcher is and new limits to protect Ingester query path against expensive unoptimized regex matchers. #7194 #7210 * [ENHANCEMENT] Compactor: Add partition group creation time to visit marker. #7217 +* [BUGFIX] Distributor: If remote write v2 is disabled, explicitly return HTTP 415 (Unsupported Media Type) for Remote Write V2 requests instead of attempting to parse them as V1. #7238 * [BUGFIX] Ring: Change DynamoDB KV to retry indefinitely for WatchKey. #7088 * [BUGFIX] Ruler: Add XFunctions validation support. #7111 * [BUGFIX] Querier: propagate Prometheus info annotations in protobuf responses. #7132 From 8e67879b24553eeb48a69ca9187e62bf84f260c8 Mon Sep 17 00:00:00 2001 From: SungJin1212 Date: Sun, 1 Feb 2026 20:34:25 +0900 Subject: [PATCH 5/5] fix error message Signed-off-by: SungJin1212 --- integration/remote_write_v2_test.go | 2 +- pkg/util/push/push.go | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/integration/remote_write_v2_test.go b/integration/remote_write_v2_test.go index 7b00e0eb7a2..d640155f7df 100644 --- a/integration/remote_write_v2_test.go +++ b/integration/remote_write_v2_test.go @@ -194,7 +194,7 @@ func TestIngest_SenderSendPRW2_DistributorNotAllowPRW2(t *testing.T) { symbols1, series, _ := e2e.GenerateSeriesV2("test_series", now, prompb.Label{Name: "job", Value: "test"}, prompb.Label{Name: "foo", Value: "bar"}) _, err = c.PushV2(symbols1, series) require.Error(t, err) - require.Contains(t, err.Error(), "server returned HTTP status 415 Unsupported Media Type: Remote Write V2 is disabled") + require.Contains(t, err.Error(), "io.prometheus.write.v2.Request protobuf message is not accepted by this server; only accepts prometheus.WriteRequest") // sample result, err := c.Query("test_series", now) diff --git a/pkg/util/push/push.go b/pkg/util/push/push.go index 05e59f4b1c0..abb4103a97f 100644 --- a/pkg/util/push/push.go +++ b/pkg/util/push/push.go @@ -170,7 +170,8 @@ func Handler(remoteWrite2Enabled bool, maxRecvMsgSize int, overrides *validation handlePRW1() case remote.WriteV2MessageType: if !remoteWrite2Enabled { - http.Error(w, "Remote Write V2 is disabled", http.StatusUnsupportedMediaType) + errMsg := fmt.Sprintf("%v protobuf message is not accepted by this server; only accepts %v", msgType, remote.WriteV1MessageType) + http.Error(w, errMsg, http.StatusUnsupportedMediaType) return } handlePRW2()