Adding Observability Pipelines Secret Management feature support (#3210)

Co-authored-by: ci.datadog-api-spec <packages@datadoghq.com>

Author: api-clients-generation-pipeline[bot]

.generator/schemas/v2/openapi.yaml

@@ -17847,6 +17847,13 @@ datadog\_api\_client.v2.model.observability\_pipeline\_elasticsearch\_destinatio
    :members:
    :show-inheritance:
 
+datadog\_api\_client.v2.model.observability\_pipeline\_elasticsearch\_destination\_auth module
+----------------------------------------------------------------------------------------------
+
+.. automodule:: datadog_api_client.v2.model.observability_pipeline_elasticsearch_destination_auth
+   :members:
+   :show-inheritance:
+
 datadog\_api\_client.v2.model.observability\_pipeline\_elasticsearch\_destination\_data\_stream module
 ------------------------------------------------------------------------------------------------------
 

docs/datadog_api_client.v2.model.rst

@@ -0,0 +1,81 @@
+"""
+Validate an observability pipeline with destination secret key returns "OK" response
+"""
+
+from datadog_api_client import ApiClient, Configuration
+from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
+from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
+from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
+    ObservabilityPipelineConfigProcessorGroup,
+)
+from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
+from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source import (
+    ObservabilityPipelineDatadogAgentSource,
+)
+from datadog_api_client.v2.model.observability_pipeline_datadog_agent_source_type import (
+    ObservabilityPipelineDatadogAgentSourceType,
+)
+from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor
+from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import (
+    ObservabilityPipelineFilterProcessorType,
+)
+from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec
+from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData
+from datadog_api_client.v2.model.observability_pipeline_sumo_logic_destination import (
+    ObservabilityPipelineSumoLogicDestination,
+)
+from datadog_api_client.v2.model.observability_pipeline_sumo_logic_destination_type import (
+    ObservabilityPipelineSumoLogicDestinationType,
+)
+
+body = ObservabilityPipelineSpec(
+    data=ObservabilityPipelineSpecData(
+        attributes=ObservabilityPipelineDataAttributes(
+            config=ObservabilityPipelineConfig(
+                destinations=[
+                    ObservabilityPipelineSumoLogicDestination(
+                        id="sumo-logic-destination",
+                        inputs=[
+                            "my-processor-group",
+                        ],
+                        type=ObservabilityPipelineSumoLogicDestinationType.SUMO_LOGIC,
+                        endpoint_url_key="SUMO_LOGIC_ENDPOINT_URL",
+                    ),
+                ],
+                processor_groups=[
+                    ObservabilityPipelineConfigProcessorGroup(
+                        enabled=True,
+                        id="my-processor-group",
+                        include="service:my-service",
+                        inputs=[
+                            "datadog-agent-source",
+                        ],
+                        processors=[
+                            ObservabilityPipelineFilterProcessor(
+                                enabled=True,
+                                id="filter-processor",
+                                include="status:error",
+                                type=ObservabilityPipelineFilterProcessorType.FILTER,
+                            ),
+                        ],
+                    ),
+                ],
+                sources=[
+                    ObservabilityPipelineDatadogAgentSource(
+                        id="datadog-agent-source",
+                        type=ObservabilityPipelineDatadogAgentSourceType.DATADOG_AGENT,
+                    ),
+                ],
+            ),
+            name="Pipeline with Secret Key",
+        ),
+        type="pipelines",
+    ),
+)
+
+configuration = Configuration()
+with ApiClient(configuration) as api_client:
+    api_instance = ObservabilityPipelinesApi(api_client)
+    response = api_instance.validate_pipeline(body=body)
+
+    print(response)

examples/v2/observability-pipelines/ValidatePipeline_1330454428.py

@@ -0,0 +1,87 @@
+"""
+Validate an observability pipeline with source secret key returns "OK" response
+"""
+
+from datadog_api_client import ApiClient, Configuration
+from datadog_api_client.v2.api.observability_pipelines_api import ObservabilityPipelinesApi
+from datadog_api_client.v2.model.observability_pipeline_config import ObservabilityPipelineConfig
+from datadog_api_client.v2.model.observability_pipeline_config_processor_group import (
+    ObservabilityPipelineConfigProcessorGroup,
+)
+from datadog_api_client.v2.model.observability_pipeline_data_attributes import ObservabilityPipelineDataAttributes
+from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination import (
+    ObservabilityPipelineDatadogLogsDestination,
+)
+from datadog_api_client.v2.model.observability_pipeline_datadog_logs_destination_type import (
+    ObservabilityPipelineDatadogLogsDestinationType,
+)
+from datadog_api_client.v2.model.observability_pipeline_decoding import ObservabilityPipelineDecoding
+from datadog_api_client.v2.model.observability_pipeline_filter_processor import ObservabilityPipelineFilterProcessor
+from datadog_api_client.v2.model.observability_pipeline_filter_processor_type import (
+    ObservabilityPipelineFilterProcessorType,
+)
+from datadog_api_client.v2.model.observability_pipeline_http_client_source import ObservabilityPipelineHttpClientSource
+from datadog_api_client.v2.model.observability_pipeline_http_client_source_auth_strategy import (
+    ObservabilityPipelineHttpClientSourceAuthStrategy,
+)
+from datadog_api_client.v2.model.observability_pipeline_http_client_source_type import (
+    ObservabilityPipelineHttpClientSourceType,
+)
+from datadog_api_client.v2.model.observability_pipeline_spec import ObservabilityPipelineSpec
+from datadog_api_client.v2.model.observability_pipeline_spec_data import ObservabilityPipelineSpecData
+
+body = ObservabilityPipelineSpec(
+    data=ObservabilityPipelineSpecData(
+        attributes=ObservabilityPipelineDataAttributes(
+            config=ObservabilityPipelineConfig(
+                destinations=[
+                    ObservabilityPipelineDatadogLogsDestination(
+                        id="datadog-logs-destination",
+                        inputs=[
+                            "my-processor-group",
+                        ],
+                        type=ObservabilityPipelineDatadogLogsDestinationType.DATADOG_LOGS,
+                    ),
+                ],
+                processor_groups=[
+                    ObservabilityPipelineConfigProcessorGroup(
+                        enabled=True,
+                        id="my-processor-group",
+                        include="service:my-service",
+                        inputs=[
+                            "http-client-source",
+                        ],
+                        processors=[
+                            ObservabilityPipelineFilterProcessor(
+                                enabled=True,
+                                id="filter-processor",
+                                include="status:error",
+                                type=ObservabilityPipelineFilterProcessorType.FILTER,
+                            ),
+                        ],
+                    ),
+                ],
+                sources=[
+                    ObservabilityPipelineHttpClientSource(
+                        id="http-client-source",
+                        type=ObservabilityPipelineHttpClientSourceType.HTTP_CLIENT,
+                        decoding=ObservabilityPipelineDecoding.DECODE_BYTES,
+                        scrape_interval_secs=15,
+                        scrape_timeout_secs=5,
+                        auth_strategy=ObservabilityPipelineHttpClientSourceAuthStrategy.BEARER,
+                        token_key="HTTP_CLIENT_TOKEN",
+                    ),
+                ],
+            ),
+            name="Pipeline with Source Secret",
+        ),
+        type="pipelines",
+    ),
+)
+
+configuration = Configuration()
+with ApiClient(configuration) as api_client:
+    api_instance = ObservabilityPipelinesApi(api_client)
+    response = api_instance.validate_pipeline(body=body)
+
+    print(response)

examples/v2/observability-pipelines/ValidatePipeline_2899320203.py

@@ -36,6 +36,7 @@ def openapi_types(_):
         return {
             "blob_prefix": (str,),
             "buffer": (ObservabilityPipelineBufferOptions,),
+            "connection_string_key": (str,),
             "container_name": (str,),
             "id": (str,),
             "inputs": ([str],),
@@ -45,6 +46,7 @@ def openapi_types(_):
     attribute_map = {
         "blob_prefix": "blob_prefix",
         "buffer": "buffer",
+        "connection_string_key": "connection_string_key",
         "container_name": "container_name",
         "id": "id",
         "inputs": "inputs",
@@ -65,6 +67,7 @@ def __init__(
             ObservabilityPipelineMemoryBufferSizeOptions,
             UnsetType,
         ] = unset,
+        connection_string_key: Union[str, UnsetType] = unset,
         **kwargs,
     ):
         """
@@ -78,6 +81,9 @@ def __init__(
         :param buffer: Configuration for buffer settings on destination components.
         :type buffer: ObservabilityPipelineBufferOptions, optional
 
+        :param connection_string_key: Name of the environment variable or secret that holds the Azure Storage connection string.
+        :type connection_string_key: str, optional
+
         :param container_name: The name of the Azure Blob Storage container to store logs in.
         :type container_name: str
 
@@ -94,6 +100,8 @@ def __init__(
             kwargs["blob_prefix"] = blob_prefix
         if buffer is not unset:
             kwargs["buffer"] = buffer
+        if connection_string_key is not unset:
+            kwargs["connection_string_key"] = connection_string_key
         super().__init__(kwargs)
 
         self_.container_name = container_name

src/datadog_api_client/v2/model/azure_storage_destination.py

@@ -36,6 +36,8 @@ def openapi_types(_):
         return {
             "buffer": (ObservabilityPipelineBufferOptions,),
             "client_id": (str,),
+            "client_secret_key": (str,),
+            "dce_uri_key": (str,),
             "dcr_immutable_id": (str,),
             "id": (str,),
             "inputs": ([str],),
@@ -47,6 +49,8 @@ def openapi_types(_):
     attribute_map = {
         "buffer": "buffer",
         "client_id": "client_id",
+        "client_secret_key": "client_secret_key",
+        "dce_uri_key": "dce_uri_key",
         "dcr_immutable_id": "dcr_immutable_id",
         "id": "id",
         "inputs": "inputs",
@@ -71,6 +75,8 @@ def __init__(
             ObservabilityPipelineMemoryBufferSizeOptions,
             UnsetType,
         ] = unset,
+        client_secret_key: Union[str, UnsetType] = unset,
+        dce_uri_key: Union[str, UnsetType] = unset,
         **kwargs,
     ):
         """
@@ -84,6 +90,12 @@ def __init__(
         :param client_id: Azure AD client ID used for authentication.
         :type client_id: str
 
+        :param client_secret_key: Name of the environment variable or secret that holds the Azure AD client secret.
+        :type client_secret_key: str, optional
+
+        :param dce_uri_key: Name of the environment variable or secret that holds the Data Collection Endpoint (DCE) URI.
+        :type dce_uri_key: str, optional
+
         :param dcr_immutable_id: The immutable ID of the Data Collection Rule (DCR).
         :type dcr_immutable_id: str
 
@@ -104,6 +116,10 @@ def __init__(
         """
         if buffer is not unset:
             kwargs["buffer"] = buffer
+        if client_secret_key is not unset:
+            kwargs["client_secret_key"] = client_secret_key
+        if dce_uri_key is not unset:
+            kwargs["dce_uri_key"] = dce_uri_key
         super().__init__(kwargs)
 
         self_.client_id = client_id

src/datadog_api_client/v2/model/microsoft_sentinel_destination.py

@@ -31,13 +31,15 @@ def openapi_types(_):
         )
 
         return {
+            "address_key": (str,),
             "auth": (ObservabilityPipelineAwsAuth,),
             "id": (str,),
             "tls": (ObservabilityPipelineTls,),
             "type": (ObservabilityPipelineAmazonDataFirehoseSourceType,),
         }
 
     attribute_map = {
+        "address_key": "address_key",
         "auth": "auth",
         "id": "id",
         "tls": "tls",
@@ -48,6 +50,7 @@ def __init__(
         self_,
         id: str,
         type: ObservabilityPipelineAmazonDataFirehoseSourceType,
+        address_key: Union[str, UnsetType] = unset,
         auth: Union[ObservabilityPipelineAwsAuth, UnsetType] = unset,
         tls: Union[ObservabilityPipelineTls, UnsetType] = unset,
         **kwargs,
@@ -57,6 +60,9 @@ def __init__(
 
         **Supported pipeline types:** logs
 
+        :param address_key: Name of the environment variable or secret that holds the Firehose delivery stream address.
+        :type address_key: str, optional
+
         :param auth: AWS authentication credentials used for accessing AWS services such as S3.
             If omitted, the system’s default credentials are used (for example, the IAM role and environment variables).
         :type auth: ObservabilityPipelineAwsAuth, optional
@@ -70,6 +76,8 @@ def __init__(
         :param type: The source type. The value should always be ``amazon_data_firehose``.
         :type type: ObservabilityPipelineAmazonDataFirehoseSourceType
         """
+        if address_key is not unset:
+            kwargs["address_key"] = address_key
         if auth is not unset:
             kwargs["auth"] = auth
         if tls is not unset:

src/datadog_api_client/v2/model/observability_pipeline_amazon_data_firehose_source.py

@@ -36,6 +36,7 @@ def openapi_types(_):
             "region": (str,),
             "tls": (ObservabilityPipelineTls,),
             "type": (ObservabilityPipelineAmazonS3SourceType,),
+            "url_key": (str,),
         }
 
     attribute_map = {
@@ -44,6 +45,7 @@ def openapi_types(_):
         "region": "region",
         "tls": "tls",
         "type": "type",
+        "url_key": "url_key",
     }
 
     def __init__(
@@ -53,6 +55,7 @@ def __init__(
         type: ObservabilityPipelineAmazonS3SourceType,
         auth: Union[ObservabilityPipelineAwsAuth, UnsetType] = unset,
         tls: Union[ObservabilityPipelineTls, UnsetType] = unset,
+        url_key: Union[str, UnsetType] = unset,
         **kwargs,
     ):
         """
@@ -76,11 +79,16 @@ def __init__(
 
         :param type: The source type. Always ``amazon_s3``.
         :type type: ObservabilityPipelineAmazonS3SourceType
+
+        :param url_key: Name of the environment variable or secret that holds the S3 bucket URL.
+        :type url_key: str, optional
         """
         if auth is not unset:
             kwargs["auth"] = auth
         if tls is not unset:
             kwargs["tls"] = tls
+        if url_key is not unset:
+            kwargs["url_key"] = url_key
         super().__init__(kwargs)
 
         self_.id = id