diff --git a/.generator/schemas/v2/openapi.yaml b/.generator/schemas/v2/openapi.yaml index 181237b18f..ad75118bfd 100644 --- a/.generator/schemas/v2/openapi.yaml +++ b/.generator/schemas/v2/openapi.yaml @@ -39514,6 +39514,8 @@ components: **Supported pipeline types:** logs' properties: + cache: + $ref: '#/components/schemas/ObservabilityPipelineDedupeProcessorCache' display_name: $ref: '#/components/schemas/ObservabilityPipelineComponentDisplayName' enabled: @@ -39551,6 +39553,19 @@ components: type: object x-pipeline-types: - logs + ObservabilityPipelineDedupeProcessorCache: + description: Configuration for the cache used to detect duplicates. + properties: + num_events: + description: The number of events to cache for duplicate detection. + example: 5000 + format: int64 + maximum: 1000000000 + minimum: 1 + type: integer + required: + - num_events + type: object ObservabilityPipelineDedupeProcessorMode: description: The deduplication mode to apply to the fields. enum: diff --git a/docs/datadog_api_client.v2.model.rst b/docs/datadog_api_client.v2.model.rst index a359f6c443..6c0d5099a1 100644 --- a/docs/datadog_api_client.v2.model.rst +++ b/docs/datadog_api_client.v2.model.rst @@ -17147,6 +17147,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor module :members: :show-inheritance: +datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor\_cache module +-------------------------------------------------------------------------------------- + +.. automodule:: datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache + :members: + :show-inheritance: + datadog\_api\_client.v2.model.observability\_pipeline\_dedupe\_processor\_mode module ------------------------------------------------------------------------------------- diff --git a/examples/v2/observability-pipelines/CreatePipeline_3363445359.py b/examples/v2/observability-pipelines/CreatePipeline_3363445359.py new file mode 100644 index 0000000000..4dafcf3217 --- /dev/null +++ b/examples/v2/observability-pipelines/CreatePipeline_3363445359.py @@ -0,0 +1,93 @@ +""" +Create a pipeline with dedupe processor with cache returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( + ObservabilityPipelineDatadogAgentSource, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import ( + ObservabilityPipelineDatadogAgentSourceType, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import ( + ObservabilityPipelineDatadogLogsDestination, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import ( + ObservabilityPipelineDatadogLogsDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import ( + ObservabilityPipelineDedupeProcessorCache, +) +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import ( + ObservabilityPipelineDedupeProcessorMode, +) +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_type import ( + ObservabilityPipelineDedupeProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineDatadogLogsDestination( + id="datadog-logs-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "datadog-agent-source", + ], + processors=[ + ObservabilityPipelineDedupeProcessor( + enabled=True, + id="dedupe-processor", + include="service:my-service", + type=ObservabilityPipelineDedupeProcessorType.DEDUPE, + fields=[ + "message", + ], + mode=ObservabilityPipelineDedupeProcessorMode.MATCH, + cache=ObservabilityPipelineDedupeProcessorCache( + num_events=5000, + ), + ), + ], + ), + ], + sources=[ + ObservabilityPipelineDatadogAgentSource( + id="datadog-agent-source", + type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT, + ), + ], + ), + name="Pipeline with Dedupe Cache", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.create_pipeline(body=body) + + print(response) diff --git a/examples/v2/observability-pipelines/CreatePipeline_581245895.py b/examples/v2/observability-pipelines/CreatePipeline_581245895.py new file mode 100644 index 0000000000..031e924bf5 --- /dev/null +++ b/examples/v2/observability-pipelines/CreatePipeline_581245895.py @@ -0,0 +1,87 @@ +""" +Create a pipeline with dedupe processor without cache returns "OK" response +""" + +from datadog_api_client import ApiClient, Configuration +from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi +from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig +from datadog_api_client.v2.model.observability_pipeline_config_processor_group import ( + ObservabilityPipelineConfigProcessorGroup, +) +from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import ( + ObservabilityPipelineDatadogAgentSource, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import ( + ObservabilityPipelineDatadogAgentSourceType, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import ( + ObservabilityPipelineDatadogLogsDestination, +) +from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import ( + ObservabilityPipelineDatadogLogsDestinationType, +) +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import ( + ObservabilityPipelineDedupeProcessorMode, +) +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_type import ( + ObservabilityPipelineDedupeProcessorType, +) +from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec +from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData + +body = ObservabilityPipelineSpec( + data=ObservabilityPipelineSpecData( + attributes=ObservabilityPipelineDataAttributes( + config=ObservabilityPipelineConfig( + destinations=[ + ObservabilityPipelineDatadogLogsDestination( + id="datadog-logs-destination", + inputs=[ + "my-processor-group", + ], + type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS, + ), + ], + processor_groups=[ + ObservabilityPipelineConfigProcessorGroup( + enabled=True, + id="my-processor-group", + include="service:my-service", + inputs=[ + "datadog-agent-source", + ], + processors=[ + ObservabilityPipelineDedupeProcessor( + enabled=True, + id="dedupe-processor", + include="service:my-service", + type=ObservabilityPipelineDedupeProcessorType.DEDUPE, + fields=[ + "message", + ], + mode=ObservabilityPipelineDedupeProcessorMode.MATCH, + ), + ], + ), + ], + sources=[ + ObservabilityPipelineDatadogAgentSource( + id="datadog-agent-source", + type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT, + ), + ], + ), + name="Pipeline with Dedupe No Cache", + ), + type="pipelines", + ), +) + +configuration = Configuration() +with ApiClient(configuration) as api_client: + api_instance = ObservabilityPipelinesApi(api_client) + response = api_instance.create_pipeline(body=body) + + print(response) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py index ec393b59d6..aa9c6e5453 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_config_processor_item.py @@ -48,6 +48,9 @@ def __init__(self, **kwargs): :param mode: The processing mode. :type mode: ObservabilityPipelineDatadogTagsProcessorMode + :param cache: Configuration for the cache used to detect duplicates. + :type cache: ObservabilityPipelineDedupeProcessorCache, optional + :param file: Defines a static enrichment table loaded from a CSV file. :type file: ObservabilityPipelineEnrichmentTableFile, optional diff --git a/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py index f0e3f6dd0a..ff589ff86c 100644 --- a/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py +++ b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor.py @@ -14,6 +14,9 @@ if TYPE_CHECKING: + from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import ( + ObservabilityPipelineDedupeProcessorCache, + ) from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import ( ObservabilityPipelineDedupeProcessorMode, ) @@ -25,6 +28,9 @@ class ObservabilityPipelineDedupeProcessor(ModelNormal): @cached_property def openapi_types(_): + from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import ( + ObservabilityPipelineDedupeProcessorCache, + ) from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import ( ObservabilityPipelineDedupeProcessorMode, ) @@ -33,6 +39,7 @@ def openapi_types(_): ) return { + "cache": (ObservabilityPipelineDedupeProcessorCache,), "display_name": (str,), "enabled": (bool,), "fields": ([str],), @@ -43,6 +50,7 @@ def openapi_types(_): } attribute_map = { + "cache": "cache", "display_name": "display_name", "enabled": "enabled", "fields": "fields", @@ -60,6 +68,7 @@ def __init__( include: str, mode: ObservabilityPipelineDedupeProcessorMode, type: ObservabilityPipelineDedupeProcessorType, + cache: Union[ObservabilityPipelineDedupeProcessorCache, UnsetType] = unset, display_name: Union[str, UnsetType] = unset, **kwargs, ): @@ -68,6 +77,9 @@ def __init__( **Supported pipeline types:** logs + :param cache: Configuration for the cache used to detect duplicates. + :type cache: ObservabilityPipelineDedupeProcessorCache, optional + :param display_name: The display name for a component. :type display_name: str, optional @@ -89,6 +101,8 @@ def __init__( :param type: The processor type. The value should always be ``dedupe``. :type type: ObservabilityPipelineDedupeProcessorType """ + if cache is not unset: + kwargs["cache"] = cache if display_name is not unset: kwargs["display_name"] = display_name super().__init__(kwargs) diff --git a/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor_cache.py b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor_cache.py new file mode 100644 index 0000000000..9ccf30dc13 --- /dev/null +++ b/src/datadog_api_client/v2/model/observability_pipeline_dedupe_processor_cache.py @@ -0,0 +1,40 @@ +# Unless explicitly stated otherwise all files in this repository are licensed under the Apache-2.0 License. +# This product includes software developed at Datadog (https://www.datadoghq.com/). +# Copyright 2019-Present Datadog, Inc. +from __future__ import annotations + + +from datadog_api_client.model_utils import ( + ModelNormal, + cached_property, +) + + +class ObservabilityPipelineDedupeProcessorCache(ModelNormal): + validations = { + "num_events": { + "inclusive_maximum": 1000000000, + "inclusive_minimum": 1, + }, + } + + @cached_property + def openapi_types(_): + return { + "num_events": (int,), + } + + attribute_map = { + "num_events": "num_events", + } + + def __init__(self_, num_events: int, **kwargs): + """ + Configuration for the cache used to detect duplicates. + + :param num_events: The number of events to cache for duplicate detection. + :type num_events: int + """ + super().__init__(kwargs) + + self_.num_events = num_events diff --git a/src/datadog_api_client/v2/models/__init__.py b/src/datadog_api_client/v2/models/__init__.py index f86d1dff08..ae78d8b84b 100644 --- a/src/datadog_api_client/v2/models/__init__.py +++ b/src/datadog_api_client/v2/models/__init__.py @@ -3296,6 +3296,9 @@ ) from datadog_api_client.v2.model.observability_pipeline_decoding import ObservabilityPipelineDecoding from datadog_api_client.v2.model.observability_pipeline_dedupe_processor import ObservabilityPipelineDedupeProcessor +from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_cache import ( + ObservabilityPipelineDedupeProcessorCache, +) from datadog_api_client.v2.model.observability_pipeline_dedupe_processor_mode import ( ObservabilityPipelineDedupeProcessorMode, ) @@ -8464,6 +8467,7 @@ "ObservabilityPipelineDatadogTagsProcessorType", "ObservabilityPipelineDecoding", "ObservabilityPipelineDedupeProcessor", + "ObservabilityPipelineDedupeProcessorCache", "ObservabilityPipelineDedupeProcessorMode", "ObservabilityPipelineDedupeProcessorType", "ObservabilityPipelineDiskBufferOptions", diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.frozen new file mode 100644 index 0000000000..0d00ad61ad --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-02-09T09:53:31.840Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.yaml new file mode 100644 index 0000000000..3e46d47934 --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_with_cache_returns_ok_response.yaml @@ -0,0 +1,40 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"cache":{"num_events":5000},"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Pipeline + with Dedupe Cache"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines + response: + body: + string: '{"data":{"id":"31a750dc-059d-11f1-a2a8-da7ad0900002","type":"pipelines","attributes":{"name":"Pipeline + with Dedupe Cache","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"cache":{"num_events":5000},"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"cache":{"num_events":5000},"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 201 + message: Created +- request: + body: null + headers: + accept: + - '*/*' + method: DELETE + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/31a750dc-059d-11f1-a2a8-da7ad0900002 + response: + body: + string: '' + headers: + content-type: + - application/json + status: + code: 204 + message: No Content +version: 1 diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.frozen b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.frozen new file mode 100644 index 0000000000..9dbed52a0e --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.frozen @@ -0,0 +1 @@ +2026-02-09T09:53:33.945Z \ No newline at end of file diff --git a/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.yaml b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.yaml new file mode 100644 index 0000000000..44fb26879f --- /dev/null +++ b/tests/v2/cassettes/test_scenarios/test_create_a_pipeline_with_dedupe_processor_without_cache_returns_ok_response.yaml @@ -0,0 +1,40 @@ +interactions: +- request: + body: '{"data":{"attributes":{"config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]},"name":"Pipeline + with Dedupe No Cache"},"type":"pipelines"}}' + headers: + accept: + - application/json + content-type: + - application/json + method: POST + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines + response: + body: + string: '{"data":{"id":"3280ccb8-059d-11f1-a2aa-da7ad0900002","type":"pipelines","attributes":{"name":"Pipeline + with Dedupe No Cache","config":{"destinations":[{"id":"datadog-logs-destination","inputs":["my-processor-group"],"type":"datadog_logs"}],"processor_groups":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"processors":[{"enabled":true,"id":"my-processor-group","include":"service:my-service","inputs":["datadog-agent-source"],"processors":[{"enabled":true,"fields":["message"],"id":"dedupe-processor","include":"service:my-service","mode":"match","type":"dedupe"}]}],"sources":[{"id":"datadog-agent-source","type":"datadog_agent"}]}}}} + + ' + headers: + content-type: + - application/vnd.api+json + status: + code: 201 + message: Created +- request: + body: null + headers: + accept: + - '*/*' + method: DELETE + uri: https://api.datadoghq.com/api/v2/obs-pipelines/pipelines/3280ccb8-059d-11f1-a2aa-da7ad0900002 + response: + body: + string: '' + headers: + content-type: + - application/json + status: + code: 204 + message: No Content +version: 1 diff --git a/tests/v2/features/observability_pipelines.feature b/tests/v2/features/observability_pipelines.feature index 07e0c8ea02..725b9e425a 100644 --- a/tests/v2/features/observability_pipelines.feature +++ b/tests/v2/features/observability_pipelines.feature @@ -35,6 +35,24 @@ Feature: Observability Pipelines And the response "data.attributes.config.processor_groups" has length 1 And the response "data.attributes.config.destinations" has length 1 + @team:DataDog/observability-pipelines + Scenario: Create a pipeline with dedupe processor with cache returns "OK" response + Given new "CreatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "dedupe-processor", "include": "service:my-service", "type": "dedupe", "fields": ["message"], "mode": "match", "cache": {"num_events": 5000}}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Dedupe Cache"}, "type": "pipelines"}} + When the request is sent + Then the response status is 201 OK + And the response "data.attributes.config.processor_groups[0].processors[0].type" is equal to "dedupe" + And the response "data.attributes.config.processor_groups[0].processors[0].cache.num_events" is equal to 5000 + + @team:DataDog/observability-pipelines + Scenario: Create a pipeline with dedupe processor without cache returns "OK" response + Given new "CreatePipeline" request + And body with value {"data": {"attributes": {"config": {"destinations": [{"id": "datadog-logs-destination", "inputs": ["my-processor-group"], "type": "datadog_logs"}], "processor_groups": [{"enabled": true, "id": "my-processor-group", "include": "service:my-service", "inputs": ["datadog-agent-source"], "processors": [{"enabled": true, "id": "dedupe-processor", "include": "service:my-service", "type": "dedupe", "fields": ["message"], "mode": "match"}]}], "sources": [{"id": "datadog-agent-source", "type": "datadog_agent"}]}, "name": "Pipeline with Dedupe No Cache"}, "type": "pipelines"}} + When the request is sent + Then the response status is 201 OK + And the response "data.attributes.config.processor_groups[0].processors[0].type" is equal to "dedupe" + And the response "data.attributes.config.processor_groups[0].processors[0].fields[0]" is equal to "message" + @generated @skip @team:DataDog/observability-pipelines Scenario: Delete a pipeline returns "Conflict" response Given new "DeletePipeline" request