diff --git a/docs-data/property-overrides.json b/docs-data/property-overrides.json index ff2493fc8d..b22cd6c0b9 100644 --- a/docs-data/property-overrides.json +++ b/docs-data/property-overrides.json @@ -778,7 +778,8 @@ "description": "Set the default storage mode for new topics. This value applies to any topic created without an explicit <> setting (that is, when the topic's `redpanda.storage.mode` is `unset`).\n\nAccepted values:\n\n* `unset`: Defer to the legacy <> and <> topic properties for Tiered Storage configuration.\n* `local`: Store data only on local disks, with no object storage involvement.\nifndef::env-cloud[]\n* `tiered`: Store data on local disks and replicate it to object storage using xref:manage:tiered-storage.adoc[Tiered Storage]. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`.\n* `cloud`: Store data primarily in object storage using xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics].\nendif::[]\nifdef::env-cloud[]\n* `tiered`: Store data on local disks and replicate it to object storage using Tiered Storage. Equivalent to setting `redpanda.remote.read` and `redpanda.remote.write` to `true`.\n* `cloud`: Store data primarily in object storage using Cloud Topics.\nendif::[]", "related_topics": [ "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]", - "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]", + "cloud-only: xref:develop:topics/cloud-topics.adoc[Manage Cloud Topics]" ], "config_scope": "cluster", "version": "v26.1.1" @@ -1696,6 +1697,7 @@ "description": "Enable Cloud Topic storage mode for this topic. When enabled, topic data is stored primarily in object storage with local storage used only as a write buffer.\n\nTIP: To configure storage modes with more flexibility, use <> which supports `local`, `tiered`, `cloud`, and `unset` modes.", "related_topics": [ "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "cloud-only: xref:develop:topics/cloud-topics.adoc[Cloud Topics]", "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]" ], "config_scope": "topic", @@ -1954,6 +1956,10 @@ "config_scope": "broker", "category": "redpanda" }, + "rpk_path": { + "description": "Path to RPK binary.", + "config_scope": "cluster" + }, "rps_limit_acls_and_users_operations": { "description": "Rate limit, in requests per second, for ACL and user operations on the controller. The controller processes cluster management requests (such as creating, deleting, and updating ACLs and users) through the controller log (raft0).", "config_scope": "cluster" @@ -1974,10 +1980,6 @@ "description": "Rate limit, in requests per second, for topic operations on the controller. The controller processes cluster management requests (such as creating, deleting, and updating topics and partition counts) through the controller log (raft0).", "config_scope": "cluster" }, - "rpk_path": { - "description": "Path to RPK binary.", - "config_scope": "cluster" - }, "sasl_mechanism": { "description": "The SASL mechanism to use when the HTTP Proxy client connects to the Kafka API. These credentials are used when the HTTP Proxy API listener has <>: `none` but the cluster requires authenticated access to the Kafka API.\n\nThis property specifies which individual SASL mechanism the HTTP Proxy client should use, while the cluster-wide available mechanisms are configured using the xref:reference:properties/cluster-properties.adoc#sasl_mechanisms[`sasl_mechanisms`] cluster property.\n\ninclude::shared:partial$http-proxy-ephemeral-credentials-breaking-change.adoc[]\n\nNOTE: While the cluster-wide xref:reference:properties/cluster-properties.adoc#sasl_mechanisms[`sasl_mechanisms`] property may support additional mechanisms (PLAIN, GSSAPI, OAUTHBEARER), HTTP Proxy client connections only support SCRAM mechanisms.", "related_topics": [ @@ -2251,7 +2253,7 @@ "write.caching": { "description": "The write caching mode to apply to a topic.\n\nWhen `write.caching` is set, it overrides the cluster property xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. Fsyncs follow <> and <>, whichever is reached first.", "related_topics": [ - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]", + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]", "xref:manage:tiered-storage.adoc[Tiered Storage]", "xref:reference:properties/cluster-properties.adoc#write_caching_default[`write_caching_default`]", "xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]" @@ -2261,7 +2263,7 @@ "write_caching_default": { "related_topics": [ "xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`]", - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]" + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]" ], "config_scope": "cluster", "description": "The default write caching mode to apply to user topics. Write caching acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. With `acks=all`, this provides lower latency while still ensuring that a majority of brokers acknowledge the write. \n\nFsyncs follow <> and <>, whichever is reached first.\n\nThe `write_caching_default` cluster property can be overridden with the xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`] topic property." diff --git a/docs-data/redpanda-property-changes-26.1.1-to-v26.1.1.json b/docs-data/redpanda-property-changes-26.1.1-to-v26.1.1.json new file mode 100644 index 0000000000..3bd10a9325 --- /dev/null +++ b/docs-data/redpanda-property-changes-26.1.1-to-v26.1.1.json @@ -0,0 +1,64 @@ +{ + "comparison": { + "oldVersion": "26.1.1", + "newVersion": "v26.1.1", + "timestamp": "2026-04-07T01:43:16.118Z" + }, + "summary": { + "newProperties": 0, + "changedDefaults": 3, + "changedDescriptions": 1, + "changedTypes": 1, + "deprecatedProperties": 0, + "removedDeprecatedProperties": 0, + "removedProperties": 0, + "emptyDescriptions": 2 + }, + "details": { + "newProperties": [], + "changedDefaults": [ + { + "name": "kafka_handler_latency_all", + "oldDefault": false, + "newDefault": null + }, + { + "name": "max_concurrent_producer_ids", + "oldDefault": 100000, + "newDefault": "Maximum value" + }, + { + "name": "max_transactions_per_coordinator", + "oldDefault": 10000, + "newDefault": "Maximum value" + } + ], + "changedDescriptions": [ + { + "name": "kafka_handler_latency_all", + "oldDescription": "Enable latency histograms for all Kafka API handlers. When disabled, only important handlers (produce, fetch, metadata, api_versions, offset_commit) have latency histograms.", + "newDescription": "Configuration property: kafka_handler_latency_all" + } + ], + "changedTypes": [ + { + "name": "kafka_handler_latency_all", + "oldType": "boolean", + "newType": "string" + } + ], + "deprecatedProperties": [], + "removedDeprecatedProperties": [], + "removedProperties": [], + "emptyDescriptions": [ + { + "name": "redpanda.remote.allowgaps", + "type": "boolean" + }, + { + "name": "redpanda.virtual.cluster.id", + "type": "string" + } + ] + } +} \ No newline at end of file diff --git a/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json b/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json deleted file mode 100644 index afe94c1e50..0000000000 --- a/docs-data/redpanda-property-changes-v25.3.7-to-v26.1.1-rc5.json +++ /dev/null @@ -1,853 +0,0 @@ -{ - "comparison": { - "oldVersion": "v25.3.7", - "newVersion": "v26.1.1-rc5", - "timestamp": "2026-03-30T15:12:26.249Z" - }, - "summary": { - "newProperties": 5, - "changedDefaults": 38, - "changedDescriptions": 36, - "changedTypes": 36, - "deprecatedProperties": 0, - "removedDeprecatedProperties": 47, - "removedProperties": 0, - "emptyDescriptions": 2 - }, - "details": { - "newProperties": [ - { - "name": "cloud_storage_gc_max_segments_per_run", - "type": "integer", - "default": 300, - "description": "Maximum number of segments to delete per housekeeping run. Each segment maps to up to three object keys (data, index, tx manifest), so a value of 300 produces 600 to 900 deletes plus one per spillover manifest." - }, - { - "name": "cloud_topics_disable_level_zero_gc_for_tests", - "type": "boolean", - "default": false, - "description": "Disables the level-zero garbage collector in cloud topics. This property exists to simplify testing and shouldn't be set in production." - }, - { - "name": "cloud_topics_disable_metastore_flush_loop_for_tests", - "type": "boolean", - "default": false, - "description": "Disables the metastore flush loop in cloud topics. The property exists to simplify testing of read replicas and shouldn't be set in production." - }, - { - "name": "cloud_topics_epoch_service_max_same_epoch_duration", - "type": "integer", - "default": 86400000, - "description": "The duration of time that a node can use the exact same epoch." - }, - { - "name": "schema_registry_avro_use_named_references", - "type": "object", - "description": "No description" - } - ], - "changedDefaults": [ - { - "name": "cloud_storage_prefetch_segments_max", - "oldDefault": null, - "newDefault": 3 - }, - { - "name": "cloud_topics_allow_materialization_failure", - "oldDefault": null, - "newDefault": false - }, - { - "name": "cloud_topics_compaction_interval_ms", - "oldDefault": null, - "newDefault": 30000 - }, - { - "name": "cloud_topics_compaction_key_map_memory", - "oldDefault": null, - "newDefault": 134217728 - }, - { - "name": "cloud_topics_compaction_max_object_size", - "oldDefault": null, - "newDefault": 134217728 - }, - { - "name": "cloud_topics_fetch_debounce_enabled", - "oldDefault": null, - "newDefault": true - }, - { - "name": "cloud_topics_gc_health_check_interval", - "oldDefault": null, - "newDefault": 10000 - }, - { - "name": "cloud_topics_l1_indexing_interval", - "oldDefault": null, - "newDefault": 4194304 - }, - { - "name": "cloud_topics_long_term_file_deletion_delay", - "oldDefault": null, - "newDefault": 3600000 - }, - { - "name": "cloud_topics_long_term_flush_interval", - "oldDefault": null, - "newDefault": 600000 - }, - { - "name": "cloud_topics_metastore_lsm_apply_timeout_ms", - "oldDefault": null, - "newDefault": 300000 - }, - { - "name": "cloud_topics_metastore_replication_timeout_ms", - "oldDefault": null, - "newDefault": 30000 - }, - { - "name": "cloud_topics_num_metastore_partitions", - "oldDefault": null, - "newDefault": 3 - }, - { - "name": "cloud_topics_parallel_fetch_enabled", - "oldDefault": null, - "newDefault": true - }, - { - "name": "cloud_topics_preregistered_object_ttl", - "oldDefault": null, - "newDefault": 3600000 - }, - { - "name": "cloud_topics_produce_no_pid_concurrency", - "oldDefault": null, - "newDefault": 32 - }, - { - "name": "cloud_topics_produce_write_inflight_limit", - "oldDefault": null, - "newDefault": 1024 - }, - { - "name": "cloud_topics_reconciliation_interval", - "oldDefault": 10000, - "newDefault": null - }, - { - "name": "cloud_topics_reconciliation_max_interval", - "oldDefault": null, - "newDefault": 10000 - }, - { - "name": "cloud_topics_reconciliation_max_object_size", - "oldDefault": null, - "newDefault": 83886080 - }, - { - "name": "cloud_topics_reconciliation_min_interval", - "oldDefault": null, - "newDefault": 250 - }, - { - "name": "cloud_topics_reconciliation_parallelism", - "oldDefault": null, - "newDefault": 8 - }, - { - "name": "cloud_topics_reconciliation_slowdown_blend", - "oldDefault": null, - "newDefault": 0.4 - }, - { - "name": "cloud_topics_reconciliation_speedup_blend", - "oldDefault": null, - "newDefault": 0.9 - }, - { - "name": "cloud_topics_reconciliation_target_fill_ratio", - "oldDefault": null, - "newDefault": 0.8 - }, - { - "name": "cloud_topics_upload_part_size", - "oldDefault": null, - "newDefault": 16777216 - }, - { - "name": "default_redpanda_storage_mode", - "oldDefault": null, - "newDefault": "unset" - }, - { - "name": "delete_topic_enable", - "oldDefault": null, - "newDefault": true - }, - { - "name": "internal_rpc_request_timeout_ms", - "oldDefault": null, - "newDefault": 10000 - }, - { - "name": "leader_balancer_mode", - "newDefault": "calibrated" - }, - { - "name": "log_compaction_disable_tx_batch_removal", - "newDefault": null - }, - { - "name": "log_compaction_max_priority_wait_ms", - "oldDefault": null, - "newDefault": 3600000 - }, - { - "name": "log_compaction_tx_batch_removal_enabled", - "oldDefault": false, - "newDefault": true - }, - { - "name": "nested_group_behavior", - "oldDefault": null, - "newDefault": "none" - }, - { - "name": "oidc_group_claim_path", - "oldDefault": null, - "newDefault": "$.groups" - }, - { - "name": "redpanda.storage.mode", - "oldDefault": null, - "newDefault": "unset" - }, - { - "name": "schema_registry_enable_qualified_subjects", - "oldDefault": null, - "newDefault": false - }, - { - "name": "tls_v1_2_cipher_suites", - "oldDefault": "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:AES256-GCM-SHA384:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:AES128-SHA:AES128-CCM:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:AES256-SHA:AES256-CCM", - "newDefault": "TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256:TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384:TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256:TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256" - } - ], - "changedDescriptions": [ - { - "name": "cloud_storage_prefetch_segments_max", - "oldDescription": "Configuration property: cloud_storage_prefetch_segments_max", - "newDescription": "Maximum number of small segments (size <= chunk size) to prefetch ahead during sequential reads. Set to 0 to disable cross-segment prefetch." - }, - { - "name": "cloud_topics_allow_materialization_failure", - "oldDescription": "Configuration property: cloud_topics_allow_materialization_failure", - "newDescription": "When enabled, the reconciler tolerates missing L0 extent objects (404 errors) during materialization. Failed extents are skipped, producing L1 state with empty offset ranges where deleted data was. Use this to recover partitions after accidental deletion of live extent objects." - }, - { - "name": "cloud_topics_compaction_interval_ms", - "oldDescription": "Configuration property: cloud_topics_compaction_interval_ms", - "newDescription": "How often to trigger background compaction for cloud topics." - }, - { - "name": "cloud_topics_compaction_key_map_memory", - "oldDescription": "Configuration property: cloud_topics_compaction_key_map_memory", - "newDescription": "Maximum number of bytes that may be used on each shard by cloud topics compaction key-offset maps." - }, - { - "name": "cloud_topics_compaction_max_object_size", - "oldDescription": "Configuration property: cloud_topics_compaction_max_object_size", - "newDescription": "Maximum size in bytes for L1 objects produced by cloud topics compaction." - }, - { - "name": "cloud_topics_fetch_debounce_enabled", - "oldDescription": "Configuration property: cloud_topics_fetch_debounce_enabled", - "newDescription": "Enables fetch debouncing in cloud topics. This mechanism guarantees that the broker fetches every object only once improving the performance and lowering the cost." - }, - { - "name": "cloud_topics_gc_health_check_interval", - "oldDescription": "Configuration property: cloud_topics_gc_health_check_interval", - "newDescription": "The interval at which the L0 garbage collector checks cluster health. GC will not proceed while the cluster is unhealthy." - }, - { - "name": "cloud_topics_l1_indexing_interval", - "oldDescription": "Configuration property: cloud_topics_l1_indexing_interval", - "newDescription": "The byte interval at which index entries are created within long term storage objects for cloud topics. Index entries are stored in the object metadata and enable efficient seeking by offset or timestamp within a partition. Lower values produce more index entries (better seek granularity) at the cost of a larger footer." - }, - { - "name": "cloud_topics_long_term_file_deletion_delay", - "oldDescription": "Configuration property: cloud_topics_long_term_file_deletion_delay", - "newDescription": "Delay before deleting stale long term files, allowing concurrent readers (e.g. read replica topics) to finish reading them before removal." - }, - { - "name": "cloud_topics_long_term_flush_interval", - "oldDescription": "Configuration property: cloud_topics_long_term_flush_interval", - "newDescription": "Time interval at which long term storage metadata is flushed to object storage." - }, - { - "name": "cloud_topics_metastore_lsm_apply_timeout_ms", - "oldDescription": "Configuration property: cloud_topics_metastore_lsm_apply_timeout_ms", - "newDescription": "Timeout for applying a replicated write batch to the local LSM database. This may take longer than usual when L0 compaction is behind and writes are being throttled." - }, - { - "name": "cloud_topics_metastore_replication_timeout_ms", - "oldDescription": "Configuration property: cloud_topics_metastore_replication_timeout_ms", - "newDescription": "Timeout for L1 metastore Raft replication and waiting for the STM to apply the replicated write batch." - }, - { - "name": "cloud_topics_num_metastore_partitions", - "oldDescription": "Configuration property: cloud_topics_num_metastore_partitions", - "newDescription": "Number of partitions for the cloud topics metastore topic, used to spread metastore load across the cluster. Higher values allow more parallel metadata operations but reduce the amount of work each partition can batch together. Only takes effect when the metastore topic is first created." - }, - { - "name": "cloud_topics_parallel_fetch_enabled", - "oldDescription": "Configuration property: cloud_topics_parallel_fetch_enabled", - "newDescription": "Enable parallel fetching in cloud topics. This mechanism improves the throughput by allowing the broker to download data needed by the fetch request using multiple shards." - }, - { - "name": "cloud_topics_preregistered_object_ttl", - "oldDescription": "Configuration property: cloud_topics_preregistered_object_ttl", - "newDescription": "Time-to-live for pre-registered L1 objects before they are expired." - }, - { - "name": "cloud_topics_produce_no_pid_concurrency", - "oldDescription": "Configuration property: cloud_topics_produce_no_pid_concurrency", - "newDescription": "Maximum number of concurrent raft replication requests for producers without a producer ID (idempotency disabled). Limits how many no-PID writes can proceed past the producer queue into raft simultaneously." - }, - { - "name": "cloud_topics_produce_write_inflight_limit", - "oldDescription": "Configuration property: cloud_topics_produce_write_inflight_limit", - "newDescription": "Maximum number of in-flight write requests per shard in the cloud topics write pipeline. Requests that exceed this limit are queued until a slot becomes available." - }, - { - "name": "cloud_topics_reconciliation_interval", - "oldDescription": "Time interval after which data is moved from short term storage to long term storage.", - "newDescription": "Configuration property: cloud_topics_reconciliation_interval" - }, - { - "name": "cloud_topics_reconciliation_max_interval", - "oldDescription": "Configuration property: cloud_topics_reconciliation_max_interval", - "newDescription": "Maximum reconciliation interval for adaptive scheduling." - }, - { - "name": "cloud_topics_reconciliation_max_object_size", - "oldDescription": "Configuration property: cloud_topics_reconciliation_max_object_size", - "newDescription": "Maximum size in bytes for L1 objects produced by the reconciler. With the default target fill ratio of 0.8, this gives an effective target object size of 64 MiB." - }, - { - "name": "cloud_topics_reconciliation_min_interval", - "oldDescription": "Configuration property: cloud_topics_reconciliation_min_interval", - "newDescription": "Minimum reconciliation interval for adaptive scheduling. The reconciler will not run more frequently than this." - }, - { - "name": "cloud_topics_reconciliation_parallelism", - "oldDescription": "Configuration property: cloud_topics_reconciliation_parallelism", - "newDescription": "Maximum number, per shard, of concurrent objects built by reconciliation" - }, - { - "name": "cloud_topics_reconciliation_slowdown_blend", - "oldDescription": "Configuration property: cloud_topics_reconciliation_slowdown_blend", - "newDescription": "Blend factor for slowing down reconciliation (0.0 to 1.0). Higher values mean reconciliation lowers its frequency faster when trying to find a frequency that produces well-sized objects. Generally this should be lower than the speedup blend, because reconciliation has less opportunities to adapt its frequency when it runs less frequently." - }, - { - "name": "cloud_topics_reconciliation_speedup_blend", - "oldDescription": "Configuration property: cloud_topics_reconciliation_speedup_blend", - "newDescription": "Blend factor for speeding up reconciliation (0.0 to 1.0). Higher values mean reconciliation increases its frequency faster when trying to find a frequency that produces well-sized objects." - }, - { - "name": "cloud_topics_reconciliation_target_fill_ratio", - "oldDescription": "Configuration property: cloud_topics_reconciliation_target_fill_ratio", - "newDescription": "Target fill ratio for L1 objects. The reconciler adapts its interval to produce objects at approximately this fill level (0.0 to 1.0)." - }, - { - "name": "cloud_topics_upload_part_size", - "oldDescription": "Configuration property: cloud_topics_upload_part_size", - "newDescription": "The part size in bytes used for multipart uploads. The minimum of 5 MiB is the smallest non-terminal part size allowed by cloud object storage providers." - }, - { - "name": "delete_topic_enable", - "oldDescription": "Configuration property: delete_topic_enable", - "newDescription": "Enable or disable topic deletion via the Kafka DeleteTopics API. When set to false, all topic deletion requests are rejected with error code 73 (TOPIC_DELETION_DISABLED). This is a cluster-wide safety setting that cannot be overridden by superusers. Topics in kafka_nodelete_topics are always protected regardless of this setting." - }, - { - "name": "internal_rpc_request_timeout_ms", - "oldDescription": "Configuration property: internal_rpc_request_timeout_ms", - "newDescription": "Default timeout for RPC requests between Redpanda nodes." - }, - { - "name": "leader_balancer_mode", - "oldDescription": "No description", - "newDescription": "Mode of the leader balancer optimization strategy. `calibrated` uses a heuristic that balances leaders based on replica counts per shard. `random` randomly moves leaders to reduce load on heavily-loaded shards. Legacy values `greedy_balanced_shards` and `random_hill_climbing` are treated as `calibrated`." - }, - { - "name": "log_compaction_max_priority_wait_ms", - "oldDescription": "Configuration property: log_compaction_max_priority_wait_ms", - "newDescription": "Maximum time a priority partition (for example, __consumer_offsets) can wait for compaction before preempting regular compaction." - }, - { - "name": "nested_group_behavior", - "oldDescription": "Configuration property: nested_group_behavior", - "newDescription": "Behavior for handling nested groups when extracting groups from authentication tokens. Two options are available - none and suffix. With none, the group is left alone (e.g. '/group/child/grandchild'). Suffix will extract the final component from the nested group (e.g. '/group' -> 'group' and '/group/child/grandchild' -> 'grandchild')." - }, - { - "name": "oidc_group_claim_path", - "oldDescription": "Configuration property: oidc_group_claim_path", - "newDescription": "JSON path to extract groups from the JWT payload." - }, - { - "name": "partition_autobalancing_node_autodecommission_timeout_sec", - "oldDescription": "Configuration property: partition_autobalancing_node_autodecommission_timeout_sec", - "newDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to decommission the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`." - }, - { - "name": "partition_autobalancing_node_availability_timeout_sec", - "oldDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`. ", - "newDescription": "When a node is unavailable for at least this timeout duration, it triggers Redpanda to move partitions off of the node. This property applies only when `partition_autobalancing_mode` is set to `continuous`." - }, - { - "name": "redpanda.cloud_topic.enabled", - "oldDescription": "No description", - "newDescription": "Configuration property: redpanda.cloud_topic.enabled" - }, - { - "name": "schema_registry_enable_qualified_subjects", - "oldDescription": "Configuration property: schema_registry_enable_qualified_subjects", - "newDescription": "Enable parsing of qualified subject syntax (:.context:subject). When false, subjects are treated literally, as subjects in the default context. When true, qualified syntax is parsed to extract context and subject." - } - ], - "changedTypes": [ - { - "name": "cloud_storage_prefetch_segments_max", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_allow_materialization_failure", - "oldType": "string", - "newType": "boolean" - }, - { - "name": "cloud_topics_compaction_interval_ms", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_compaction_key_map_memory", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_compaction_max_object_size", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_fetch_debounce_enabled", - "oldType": "string", - "newType": "boolean" - }, - { - "name": "cloud_topics_gc_health_check_interval", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_l1_indexing_interval", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_long_term_file_deletion_delay", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_long_term_flush_interval", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_metastore_lsm_apply_timeout_ms", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_metastore_replication_timeout_ms", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_num_metastore_partitions", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_parallel_fetch_enabled", - "oldType": "string", - "newType": "boolean" - }, - { - "name": "cloud_topics_preregistered_object_ttl", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_produce_no_pid_concurrency", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_produce_write_inflight_limit", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_reconciliation_interval", - "oldType": "integer", - "newType": "string" - }, - { - "name": "cloud_topics_reconciliation_max_interval", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_reconciliation_max_object_size", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_reconciliation_min_interval", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_reconciliation_parallelism", - "oldType": "string", - "newType": "integer" - }, - { - "name": "cloud_topics_reconciliation_slowdown_blend", - "oldType": "string", - "newType": "number" - }, - { - "name": "cloud_topics_reconciliation_speedup_blend", - "oldType": "string", - "newType": "number" - }, - { - "name": "cloud_topics_reconciliation_target_fill_ratio", - "oldType": "string", - "newType": "number" - }, - { - "name": "cloud_topics_upload_part_size", - "oldType": "string", - "newType": "integer" - }, - { - "name": "coproc_supervisor_server", - "oldType": "deprecated_property", - "newType": "object" - }, - { - "name": "dashboard_dir", - "oldType": "deprecated_property", - "newType": "object" - }, - { - "name": "delete_topic_enable", - "oldType": "string", - "newType": "boolean" - }, - { - "name": "enable_central_config", - "oldType": "deprecated_property", - "newType": "object" - }, - { - "name": "internal_rpc_request_timeout_ms", - "oldType": "string", - "newType": "integer" - }, - { - "name": "leader_balancer_mode", - "oldType": "deprecated_property", - "newType": "string" - }, - { - "name": "log_compaction_disable_tx_batch_removal", - "oldType": "deprecated_property", - "newType": "string" - }, - { - "name": "log_compaction_max_priority_wait_ms", - "oldType": "string", - "newType": "integer" - }, - { - "name": "partition_autobalancing_node_autodecommission_timeout_sec", - "oldType": "string", - "newType": "integer" - }, - { - "name": "schema_registry_enable_qualified_subjects", - "oldType": "string", - "newType": "boolean" - } - ], - "deprecatedProperties": [], - "removedDeprecatedProperties": [ - { - "name": "alter_topic_cfg_timeout_ms", - "type": "integer", - "description": "The duration, in milliseconds, that Redpanda waits for the replication of entries in the controller log when executing a request to alter topic configurations. This timeout ensures that configuration changes are replicated across the cluster before the alteration request is considered complete." - }, - { - "name": "cloud_storage_disable_metadata_consistency_checks", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "cloud_storage_reconciliation_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "coproc_max_batch_size", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "coproc_max_inflight_bytes", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "coproc_max_ingest_bytes", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "coproc_offset_flush_interval_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "create_topic_timeout_ms", - "type": "integer", - "description": "Timeout, in milliseconds, to wait for new topic creation." - }, - { - "name": "datalake_disk_space_monitor_interval", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "enable_admin_api", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "enable_coproc", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "find_coordinator_timeout_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "full_raft_configuration_recovery_pattern", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "id_allocator_replication", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_admin_topic_api_rate", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_client_group_byte_rate_quota", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_client_group_fetch_byte_rate_quota", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_memory_batch_size_estimate_for_fetch", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_quota_balancer_min_shard_throughput_bps", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_quota_balancer_min_shard_throughput_ratio", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_quota_balancer_node_period", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_quota_balancer_window", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "kafka_throughput_throttling_v2", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "log_compaction_adjacent_merge_self_compaction_count", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "log_message_timestamp_alert_after_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "log_message_timestamp_alert_before_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "max_version", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "metadata_status_wait_timeout_ms", - "type": "integer", - "description": "Maximum time to wait in metadata request for cluster health to be refreshed." - }, - { - "name": "min_version", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "node_management_operation_timeout_ms", - "type": "integer", - "description": "Timeout for executing node management operations." - }, - { - "name": "raft_max_concurrent_append_requests_per_follower", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "raft_recovery_default_read_size", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "recovery_append_timeout_ms", - "type": "integer", - "description": "Timeout for append entry requests issued while updating a stale follower." - }, - { - "name": "rm_sync_timeout_ms", - "type": "integer", - "description": "Resource manager's synchronization timeout. Specifies the maximum time for this node to wait for the internal state machine to catch up with all events written by previous leaders before rejecting a request." - }, - { - "name": "rm_violation_recovery_policy", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "schema_registry_protobuf_renderer_v2", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "seed_server_meta_topic_partitions", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "seq_table_min_size", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "target_fetch_quota_byte_rate", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "target_quota_byte_rate", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "tm_sync_timeout_ms", - "type": "integer", - "description": "Transaction manager's synchronization timeout. Maximum time to wait for internal state machine to catch up before rejecting a request." - }, - { - "name": "tm_violation_recovery_policy", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "transaction_coordinator_replication", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "tx_registry_log_capacity", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "tx_registry_sync_timeout_ms", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "use_scheduling_groups", - "type": "deprecated_property", - "description": "No description" - }, - { - "name": "wait_for_leader_timeout_ms", - "type": "integer", - "description": "Timeout to wait for leadership in metadata cache." - } - ], - "removedProperties": [], - "emptyDescriptions": [ - { - "name": "redpanda.remote.allowgaps", - "type": "boolean" - }, - { - "name": "redpanda.virtual.cluster.id", - "type": "string" - } - ] - } -} \ No newline at end of file diff --git a/modules/deploy/pages/redpanda/kubernetes/k-production-readiness.adoc b/modules/deploy/pages/redpanda/kubernetes/k-production-readiness.adoc index 01ae3cd02a..8ea242a90d 100644 --- a/modules/deploy/pages/redpanda/kubernetes/k-production-readiness.adoc +++ b/modules/deploy/pages/redpanda/kubernetes/k-production-readiness.adoc @@ -529,7 +529,7 @@ Ensure storage classes provide adequate IOPS and throughput for your workload by * Use NVMe-based storage classes for production deployments * Specify a minimum 16,000 IOPS (Input/Output Operations Per Second) * Consider provisioned IOPS where available to meet or exceed the minimum -* Enable xref:develop:config-topics.adoc#configure-write-caching[write caching] to help Redpanda perform better in environments with disks that don't meet the recommended IOPS +* Enable xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching] to help Redpanda perform better in environments with disks that don't meet the recommended IOPS * NFS (Network File System) is not supported * Test storage performance under load diff --git a/modules/develop/pages/config-topics.adoc b/modules/develop/pages/config-topics.adoc deleted file mode 100644 index a2e7117030..0000000000 --- a/modules/develop/pages/config-topics.adoc +++ /dev/null @@ -1,279 +0,0 @@ -= Manage Topics -:page-categories: Clients, Development -:description: Learn how to create topics, update topic configurations, and delete topics or records. -// tag::single-source[] - -include::develop:partial$topic-defaults.adoc[] - -== Create a topic - -Creating a topic can be as simple as specifying a name for your topic on the command line. For example, to create a topic named `xyz`, run: - -[,bash] ----- -rpk topic create xyz ----- - -ifndef::env-cloud[] -This command creates a topic named `xyz` with one partition and one replica, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. - -endif::[] - -ifdef::env-cloud[] -This command creates a topic named `xyz` with one partition and three replicas, because these are the default values set in the cluster configuration file. Replicas are copies of partitions that are distributed across different brokers, so if one broker goes down, other brokers still have a copy of the data. - -Redpanda Cloud supports 40,000 topics per cluster. - -endif::[] - -=== Choose the number of partitions - -A partition acts as a log file where topic data is written. Dividing topics into partitions allows producers to write messages in parallel and consumers to read messages in parallel. The higher the number of partitions, the greater the throughput. - -TIP: As a general rule, select a number of partitions that corresponds to the maximum number of consumers in any consumer group that will consume the data. - -For example, suppose you plan to create a consumer group with 10 consumers. To create topic `xyz` with 10 partitions, run: - -[,bash] ----- -rpk topic create xyz -p 10 ----- - -ifndef::env-cloud[] -=== Choose the replication factor - -The default replication factor in the cluster configuration is set to 1. By choosing a replication factor greater than 1, you ensure that each partition has a copy of its data on at least one other broker. One replica acts as the leader, and the other replicas are followers. - -To specify a replication factor of 3 for topic `xyz`, run: - -[,bash] ----- -rpk topic create xyz -r 3 ----- - -NOTE: The replication factor must be an odd number. Redpanda Data recommends a replication factor of 3 for most use cases. Administrators may set a minimum required replication factor for any new topic in the cluster through the cluster-level xref:reference:cluster-properties.adoc#minimum_topic_replications[`minimum_topic_replications`] property. - -TIP: If you enable xref:manage:tiered-storage.adoc[Tiered Storage] on a topic, you can then use xref:manage:topic-recovery.adoc[topic recovery] to restore data for a deleted topic. - -endif::[] - -== Update topic configurations - -After you create a topic, you can update the topic property settings for all new data written to it. For example, you can add partitions or change the cleanup policy. - -=== Add partitions - -You can assign a certain number of partitions when you create a topic, and add partitions later. For example, suppose you add brokers to your cluster, and you want to take advantage of the additional processing power. To increase the number of partitions for existing topics, run: - -[,bash] ----- -rpk topic add-partitions [TOPICS...] --num [#] ----- - -Note that `--num <#>` is the number of partitions to _add_, not the total number of partitions. - -include::develop:partial$balance-existing-topic-redistribution.adoc[] - -ifndef::env-cloud[] -=== Change the replication factor - -Suppose you create a topic with the default replication factor of 1 (which is specified in the cluster properties configuration file). Now you want to change the replication factor to 3, so you can have two backups of topic data in case a broker goes down. To set the replication factor to 3, run: - -[,bash] ----- -rpk topic alter-config [TOPICS...] --set replication.factor=3 ----- - -NOTE: The replication factor can't exceed the number of Redpanda brokers. If you try to set a replication factor greater than the number of brokers, the request is rejected. - -endif::[] - -=== Change the cleanup policy - -The cleanup policy determines how to clean up the partition log files when they reach a certain size: - -* `delete` deletes data based on age or log size. Topics retain all records until then. -* `compact` compacts the data by only keeping the latest values for each KEY. -* `compact,delete` combines both methods. - -Unlike compacted topics, which keep only the most recent message for a given key, topics configured with a `delete` cleanup policy provide a running history of all changes for those topics. - -[NOTE] -==== -include::shared:partial$tristate-behavior-change-25-3.adoc[] -==== - -include::develop:partial$topic-properties-warning.adoc[] - -For example, to change a topic's policy to `compact`, run: - -[,bash] ----- -rpk topic alter-config [TOPICS…] —-set cleanup.policy=compact ----- - -ifndef::env-cloud[] -For details on compaction in Redpanda, see xref:manage:cluster-maintenance/compaction-settings.adoc[Compaction settings]. - -endif::[] - -=== Configure write caching - -Write caching is a relaxed mode of xref:develop:produce-data/configure-producers.adoc#acksall[`acks=all`] that provides better performance at the expense of durability. It acknowledges a message as soon as it is received and acknowledged on a majority of brokers, without waiting for it to be written to disk. This provides lower latency while still ensuring that a majority of brokers acknowledge the write. - -Write caching applies to user topics. It does not apply to transactions or consumer offsets: data written in the context of a transaction and consumer offset commits is always written to disk and fsynced before being acknowledged to the client. - -ifndef::env-cloud[] -NOTE: For clusters in xref:reference:rpk/rpk-redpanda/rpk-redpanda-mode.adoc#development-mode[development mode], write caching is enabled by default. For clusters in production mode, it is disabled by default. - -endif::[] - -Only enable write caching on workloads that can tolerate some data loss in the case of multiple, simultaneous broker failures. Leaving write caching disabled safeguards your data against complete data center or availability zone failures. - -ifndef::env-cloud[] - -==== Configure at cluster level - -To enable write caching by default in all user topics, set the cluster-level property xref:reference:cluster-properties.adoc#write_caching_default[`write_caching_default`]: - -`rpk cluster config set write_caching_default=true` - -With `write_caching_default` set to true at the cluster level, Redpanda fsyncs to disk according to xref:reference:cluster-properties.adoc#raft_replica_max_pending_flush_bytes[`raft_replica_max_pending_flush_bytes`] and xref:reference:cluster-properties.adoc#raft_replica_max_flush_delay_ms[`raft_replica_max_flush_delay_ms`], whichever is reached first. - -endif::[] - -==== Configure at topic level - -To override the cluster-level setting at the topic level, set the topic-level property `write.caching`: - -`rpk topic alter-config my_topic --set write.caching=true` - -With `write.caching` enabled at the topic level, Redpanda fsyncs to disk according to `flush.ms` and `flush.bytes`, whichever is reached first. - -=== Remove a configuration setting - -You can remove a configuration that overrides the default setting, and the setting will use the default value again. For example, suppose you altered the cleanup policy to use `compact` instead of the default, `delete`. Now you want to return the policy setting to the default. To remove the configuration setting `cleanup.policy=compact`, run `rpk topic alter-config` with the `--delete` flag: - -[,bash] ----- -rpk topic alter-config [TOPICS...] --delete cleanup.policy ----- - -== List topic configuration settings - -To display all the configuration settings for a topic, run: - -[,bash] ----- -rpk topic describe -c ----- - -The `-c` flag limits the command output to just the topic configurations. This command is useful for checking the default configuration settings before you make any changes and for verifying changes after you make them. - -The following command output displays after running `rpk topic describe test-topic`, where `test-topic` was created with default settings: - -ifndef::env-cloud[] -[,bash] ----- -rpk topic describe test_topic -SUMMARY -======= -NAME test_topic -PARTITIONS 1 -REPLICAS 1 - -CONFIGS -======= -KEY VALUE SOURCE -cleanup.policy delete DYNAMIC_TOPIC_CONFIG -compression.type producer DEFAULT_CONFIG -max.message.bytes 1048576 DEFAULT_CONFIG -message.timestamp.type CreateTime DEFAULT_CONFIG -redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG -redpanda.remote.delete true DEFAULT_CONFIG -redpanda.remote.read false DEFAULT_CONFIG -redpanda.remote.write false DEFAULT_CONFIG -retention.bytes -1 DEFAULT_CONFIG -retention.local.target.bytes -1 DEFAULT_CONFIG -retention.local.target.ms 86400000 DEFAULT_CONFIG -retention.ms 604800000 DEFAULT_CONFIG -segment.bytes 1073741824 DEFAULT_CONFIG ----- - -Suppose you add two partitions, and increase the number of replicas to 3. The new command output confirms the changes in the `SUMMARY` section: - -[.no-copy] ----- -SUMMARY -======= -NAME test_topic -PARTITIONS 3 -REPLICAS 3 ----- - -endif::[] - -ifdef::env-cloud[] -[,bash] ----- -rpk topic describe test_topic -SUMMARY -======= -NAME test_topic -PARTITIONS 1 -REPLICAS 3 - -CONFIGS -======= -KEY VALUE SOURCE -cleanup.policy delete DYNAMIC_TOPIC_CONFIG -compression.type producer DEFAULT_CONFIG -max.message.bytes 20971520 DEFAULT_CONFIG -message.timestamp.type CreateTime DEFAULT_CONFIG -redpanda.datapolicy function_name: script_name: DEFAULT_CONFIG -redpanda.remote.delete true DEFAULT_CONFIG -redpanda.remote.read false DEFAULT_CONFIG -redpanda.remote.write false DEFAULT_CONFIG -retention.bytes -1 DEFAULT_CONFIG -retention.local.target.bytes -1 DEFAULT_CONFIG -retention.local.target.ms 86400000 DEFAULT_CONFIG -retention.ms 604800000 DEFAULT_CONFIG -segment.bytes 1073741824 DEFAULT_CONFIG ----- - -endif::[] - -== Delete a topic - -To delete a topic, run: - -[,bash] ----- -rpk topic delete ----- - -When a topic is deleted, its underlying data is deleted, too. - -To delete multiple topics at a time, provide a space-separated list. For example, to delete two topics named `topic1` and `topic2`, run: - -[,bash] ----- -rpk topic delete topic1 topic2 ----- - -You can also use the `-r` flag to specify one or more regular expressions; then, any topic names that match the pattern you specify are deleted. For example, to delete topics with names that start with "`f`" and end with "`r`", run: - -[,bash] ----- -rpk topic delete -r '^f.*' '.*r$' ----- - -Note that the first regular expression must start with the `^` symbol, and the last expression must end with the `$` symbol. This requirement helps prevent accidental deletions. - -include::develop:partial$delete-topic-records.adoc[] - -== Next steps - -xref:develop:produce-data/configure-producers.adoc[] - -// end::single-source[] diff --git a/modules/develop/pages/manage-topics/config-topics.adoc b/modules/develop/pages/manage-topics/config-topics.adoc index 09d5c0b191..36f412a8a3 100644 --- a/modules/develop/pages/manage-topics/config-topics.adoc +++ b/modules/develop/pages/manage-topics/config-topics.adoc @@ -1,5 +1,6 @@ = Manage Topics :page-categories: Clients, Development +:page-aliases: develop:config-topics.adoc :description: Learn how to create topics, update topic configurations, and delete topics or records. tag::single-source[] diff --git a/modules/develop/pages/produce-data/leader-pinning.adoc b/modules/develop/pages/produce-data/leader-pinning.adoc index 74cb3a0da8..323d177f96 100644 --- a/modules/develop/pages/produce-data/leader-pinning.adoc +++ b/modules/develop/pages/produce-data/leader-pinning.adoc @@ -178,8 +178,14 @@ If a higher-priority rack recovers and the topic's replication factor ensures th == Suggested reading -// TODO: Add link to Cloud Topics -// * For latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major cost driver, also consider xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] +ifndef::env-cloud[] +* For latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major cost driver, also consider xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] +endif::[] +ifdef::env-cloud[] +* For latency-tolerant, high-throughput workloads where cross-AZ networking charges are a major cost driver, also consider xref:develop:topics/cloud-topics.adoc[Cloud Topics] +endif::[] + + * xref:develop:consume-data/follower-fetching.adoc[] // end::single-source[] \ No newline at end of file diff --git a/modules/get-started/pages/licensing/overview.adoc b/modules/get-started/pages/licensing/overview.adoc index 93d0a73edc..8c50d327dc 100644 --- a/modules/get-started/pages/licensing/overview.adoc +++ b/modules/get-started/pages/licensing/overview.adoc @@ -128,6 +128,10 @@ Continuous Intra-Broker Partition Balancing is enabled by default for all new cl | Enables compliance with FIPS security standards for cryptography. | No change. +| xref:manage:security/authorization/gbac.adoc[Group-Based Access Control (GBAC)] +| Manages permissions using OIDC group memberships for ACLs and role assignments. +| ACLs with `Group:` principals cannot be created. Existing group ACLs continue to be evaluated and can be deleted. + | xref:manage:iceberg/topic-iceberg-integration.adoc[Iceberg Topics] | Enables Iceberg integration for Redpanda topics. | Topics cannot be created or modified with the `redpanda.iceberg.mode` property. diff --git a/modules/manage/pages/disaster-recovery/shadowing/overview.adoc b/modules/manage/pages/disaster-recovery/shadowing/overview.adoc index 54f1d4c6b3..66f6178920 100644 --- a/modules/manage/pages/disaster-recovery/shadowing/overview.adoc +++ b/modules/manage/pages/disaster-recovery/shadowing/overview.adoc @@ -171,7 +171,7 @@ The filtering system you configure determines the precise scope of replication a To ensure reliable disaster recovery with Shadowing: ifndef::env-cloud[] -* **Avoid write caching on source topics**: Do not shadow source topics that have xref:develop:config-topics.adoc#configure-write-caching[write caching] enabled. Write caching can result in data loss on the source cluster during broker resets, causing cluster divergence if shadow links replicate data before it's lost on the source. +* **Avoid write caching on source topics**: Do not shadow source topics that have xref:develop:manage-topics/config-topics.adoc#configure-write-caching[write caching] enabled. Write caching can result in data loss on the source cluster during broker resets, causing cluster divergence if shadow links replicate data before it's lost on the source. endif::[] * **Do not modify shadow topic properties**: Avoid modifying synced topic properties on shadow topics, as these properties automatically revert to source topic values. diff --git a/modules/manage/pages/iceberg/about-iceberg-topics.adoc b/modules/manage/pages/iceberg/about-iceberg-topics.adoc index ad771ef397..d785e11fd7 100644 --- a/modules/manage/pages/iceberg/about-iceberg-topics.adoc +++ b/modules/manage/pages/iceberg/about-iceberg-topics.adoc @@ -122,7 +122,7 @@ The link:/api/doc/cloud-controlplane/operation/operation-clusterservice_updatecl endif::[] ifndef::env-cloud[] + -When multiple clusters write to the same catalog, each cluster must use a distinct namespace to avoid table name collisions. This is especially critical for REST catalog providers that offer a single global catalog per account (such as AWS Glue), where there is no other isolation mechanism. By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. To use a unique namespace for your cluster's REST catalog integration, set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] at the same time. This property cannot be changed after you enable Iceberg topics on the cluster. +When multiple clusters write to the same catalog, each cluster must use a distinct namespace to avoid table name collisions. This is especially critical for REST catalog providers that offer a single global catalog per account (such as AWS Glue), where there is no other isolation mechanism. By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. To use a unique namespace for your cluster's REST catalog integration, also set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] when you set `iceberg_enabled`. This property cannot be changed after you enable Iceberg topics on the cluster. + [,bash] ---- diff --git a/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc index c0a58864e7..7d9699bef4 100644 --- a/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc +++ b/modules/manage/pages/iceberg/iceberg-topics-aws-glue.adoc @@ -152,7 +152,7 @@ To configure your Redpanda cluster to enable Iceberg on a topic and integrate wi . Edit your cluster configuration to set the `iceberg_enabled` property to `true`, and set the catalog integration properties listed in the example below. ifndef::env-cloud[] + -By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. Because AWS Glue provides a single catalog per account, each Redpanda cluster that writes to the same Glue catalog must use a distinct namespace to avoid table name collisions. To set a unique namespace, set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] at the same time. This property cannot be changed after Iceberg is enabled. +By default, Redpanda creates Iceberg tables in a namespace called `redpanda`. Because AWS Glue provides a single catalog per account, each Redpanda cluster that writes to the same Glue catalog must use a distinct namespace to avoid table name collisions. To set a unique namespace, also set config_ref:iceberg_default_catalog_namespace,true,properties/cluster-properties[`iceberg_default_catalog_namespace`] when you set `iceberg_enabled`. This property cannot be changed after Iceberg is enabled. + Run `rpk cluster config edit` to update these properties: + @@ -301,7 +301,12 @@ To query the table in Amazon Athena: . On the list of tables in AWS Glue Studio, click "Table data" under the *View data* column. . Click "Proceed" to be redirected to the Athena query editor. +ifndef::env-cloud[] +. In the query editor, select AwsDataCatalog as the data source, and select the `redpanda` database. If you set a custom namespace for your cluster, select that database instead of `redpanda`. +endif::[] +ifdef::env-cloud[] . In the query editor, select AwsDataCatalog as the data source, and select the `redpanda` database. +endif::[] . The SQL query editor should be pre-populated with a query that selects 10 rows from the Iceberg table. Run the query to see a preview of the table data. + [,sql] diff --git a/modules/reference/attachments/redpanda-properties-v26.1.2.json b/modules/reference/attachments/redpanda-properties-v26.1.2.json index 76ea8f7035..4ebb6bb6b6 100644 --- a/modules/reference/attachments/redpanda-properties-v26.1.2.json +++ b/modules/reference/attachments/redpanda-properties-v26.1.2.json @@ -5513,7 +5513,8 @@ "nullable": false, "related_topics": [ "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]", - "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]" + "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics]", + "cloud-only: xref:develop:topics/cloud-topics.adoc[Manage Cloud Topics]" ], "type": "string", "version": "v26.1.1", @@ -6785,10 +6786,10 @@ }, "iceberg_default_catalog_namespace": { "c_type": "ss::sstring", - "cloud_byoc_only": false, - "cloud_editable": false, + "cloud_byoc_only": true, + "cloud_editable": true, "cloud_readonly": false, - "cloud_supported": false, + "cloud_supported": true, "config_scope": "cluster", "default": [ "redpanda" @@ -11257,6 +11258,7 @@ "name": "redpanda.cloud_topic.enabled", "related_topics": [ "self-managed-only: xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics]", + "cloud-only: xref:develop:topics/cloud-topics.adoc[Cloud Topics]", "self-managed-only: xref:manage:tiered-storage.adoc[Tiered Storage]" ], "type": "string", @@ -13953,7 +13955,7 @@ "name": "write.caching", "needs_restart": false, "related_topics": [ - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]", + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]", "xref:manage:tiered-storage.adoc[Tiered Storage]", "xref:reference:properties/cluster-properties.adoc#write_caching_default[`write_caching_default`]", "xref:cluster-properties.adoc#write_caching_default[`write_caching_default`]" @@ -13983,7 +13985,7 @@ "nullable": false, "related_topics": [ "xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`]", - "xref:develop:config-topics.adoc#configure-write-caching[Write caching]" + "xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching]" ], "type": "string", "visibility": "user" diff --git a/modules/reference/partials/properties/cluster-properties.adoc b/modules/reference/partials/properties/cluster-properties.adoc index 02dbd9e589..0baf18c8ca 100644 --- a/modules/reference/partials/properties/cluster-properties.adoc +++ b/modules/reference/partials/properties/cluster-properties.adoc @@ -4990,14 +4990,20 @@ endif::[] | `tiered` -ifndef::env-cloud[] | Related topics | +ifndef::env-cloud[] * xref:manage:tiered-storage.adoc[Tiered Storage] +endif::[] +ifndef::env-cloud[] * xref:develop:manage-topics/cloud-topics.adoc[Manage Cloud Topics] endif::[] +ifdef::env-cloud[] +* xref:develop:topics/cloud-topics.adoc[Manage Cloud Topics] +endif::[] + |=== // end::redpanda-cloud[] @@ -7446,6 +7452,7 @@ endif::[] // end::redpanda-cloud[] +// tag::redpanda-cloud[] === iceberg_default_catalog_namespace ifndef::env-cloud[] @@ -7456,6 +7463,11 @@ The default namespace (database name) for Iceberg tables. All tables created by IMPORTANT: This value must be configured before enabling Iceberg and must not be changed afterward. Changing it will cause Redpanda to lose track of existing tables. +ifdef::env-cloud[] +NOTE: This property is available only in Redpanda Cloud BYOC deployments. +endif::[] + + [cols="1s,2a"] |=== | Property | Value @@ -7468,7 +7480,7 @@ IMPORTANT: This value must be configured before enabling Iceberg and must not be | Default | ifdef::env-cloud[] -Available in the Redpanda Cloud Console +Available in the Redpanda Cloud Console (editable) endif::[] ifndef::env-cloud[] `[redpanda]` @@ -7492,6 +7504,7 @@ endif::[] |=== +// end::redpanda-cloud[] // tag::redpanda-cloud[] === iceberg_default_partition_spec @@ -19836,7 +19849,7 @@ endif::[] | * xref:reference:properties/topic-properties.adoc#writecaching[`write.caching`] -* xref:develop:config-topics.adoc#configure-write-caching[Write caching] +* xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching] |=== diff --git a/modules/reference/partials/properties/topic-properties.adoc b/modules/reference/partials/properties/topic-properties.adoc index 80491397af..1fbf3161dc 100644 --- a/modules/reference/partials/properties/topic-properties.adoc +++ b/modules/reference/partials/properties/topic-properties.adoc @@ -977,11 +977,17 @@ ifndef::env-cloud[] | Yes endif::[] -ifndef::env-cloud[] | Related topics | +ifndef::env-cloud[] * xref:develop:manage-topics/cloud-topics.adoc[Cloud Topics] +endif::[] +ifdef::env-cloud[] +* xref:develop:topics/cloud-topics.adoc[Cloud Topics] +endif::[] + +ifndef::env-cloud[] * xref:manage:tiered-storage.adoc[Tiered Storage] endif::[] @@ -2192,7 +2198,7 @@ endif::[] | Related topics | -* xref:develop:config-topics.adoc#configure-write-caching[Write caching] +* xref:develop:manage-topics/config-topics.adoc#configure-write-caching[Write caching] * xref:manage:tiered-storage.adoc[Tiered Storage]