From 1e2f6400ed7923bbb01c0438c6bbf10e6088b76c Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Sun, 26 Apr 2026 19:11:22 +0000 Subject: [PATCH 1/7] feat: add `wal2json` container image This plugin supports extracting WAL changes and converting them to JSON format. Closes #77 Assisted-by: Claude Opus 4.7 Signed-off-by: solidDoWant --- wal2json/Dockerfile | 26 ++++++ wal2json/README.md | 116 +++++++++++++++++++++++++++ wal2json/metadata.hcl | 33 ++++++++ wal2json/test/chainsaw-test.yaml | 36 +++++++++ wal2json/test/check-slot-assert.yaml | 6 ++ wal2json/test/check-slot.yaml | 28 +++++++ wal2json/test/cluster-assert.yaml | 8 ++ wal2json/test/cluster.yaml | 16 ++++ 8 files changed, 269 insertions(+) create mode 100644 wal2json/Dockerfile create mode 100644 wal2json/README.md create mode 100644 wal2json/metadata.hcl create mode 100644 wal2json/test/chainsaw-test.yaml create mode 100644 wal2json/test/check-slot-assert.yaml create mode 100644 wal2json/test/check-slot.yaml create mode 100644 wal2json/test/cluster-assert.yaml create mode 100644 wal2json/test/cluster.yaml diff --git a/wal2json/Dockerfile b/wal2json/Dockerfile new file mode 100644 index 0000000..bc24cbb --- /dev/null +++ b/wal2json/Dockerfile @@ -0,0 +1,26 @@ +# SPDX-FileCopyrightText: Copyright © contributors to CloudNativePG, established as CloudNativePG a Series of LF Projects, LLC. +# SPDX-License-Identifier: Apache-2.0 + +ARG BASE=ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie +FROM $BASE AS builder + +ARG PG_MAJOR +ARG EXT_VERSION + +USER 0 + +# Install extension via `apt-get` +RUN apt-get update && apt-get install -y --no-install-recommends \ + "postgresql-${PG_MAJOR}-wal2json=${EXT_VERSION}" + +FROM scratch +ARG PG_MAJOR + +# Licenses +COPY --from=builder /usr/share/doc/postgresql-${PG_MAJOR}-wal2json/copyright /licenses/postgresql-${PG_MAJOR}-wal2json/ + +# Libraries +COPY --from=builder /usr/lib/postgresql/${PG_MAJOR}/lib/wal2json* /lib/ +COPY --from=builder /usr/lib/postgresql/${PG_MAJOR}/lib/bitcode/ /lib/bitcode/ + +USER 65532:65532 diff --git a/wal2json/README.md b/wal2json/README.md new file mode 100644 index 0000000..5f3fa63 --- /dev/null +++ b/wal2json/README.md @@ -0,0 +1,116 @@ +# wal2json + + +[wal2json](https://github.com/eulerto/wal2json) is a PostgreSQL **logical +decoding output plugin**. It reads the write-ahead log (WAL) of a database via +a logical replication slot and emits the changes as JSON, which makes it a +common building block for change data capture (CDC) pipelines (e.g. Debezium, +custom replication tooling, or audit streams). + +Because `wal2json` is a logical decoding output plugin rather than a regular +extension, it does **not** expose a `CREATE EXTENSION` object. The plugin is +loaded on demand by PostgreSQL when a replication slot is created with the +`wal2json` output plugin. + +This image provides a convenient way to deploy and manage `wal2json` with +[CloudNativePG](https://cloudnative-pg.io/). + +## Usage + +### 1. Add the wal2json extension image to your Cluster + +Define the `wal2json` extension under the `postgresql.extensions` section of +your `Cluster` resource. Logical decoding requires `wal_level` to be set to +`logical`: + +```yaml +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: cluster-wal2json +spec: + imageName: ghcr.io/cloudnative-pg/postgresql:18-minimal-trixie + instances: 1 + + storage: + size: 1Gi + + postgresql: + parameters: + wal_level: logical + extensions: + - name: wal2json + image: + # renovate: suite=trixie-pgdg depName=postgresql-18-wal2json + reference: ghcr.io/cloudnative-pg/wal2json:2.6-18-trixie +``` + +> [!NOTE] +> Unlike most extensions, `wal2json` is a logical decoding output plugin and +> does not require a `Database` resource with `CREATE EXTENSION`. The plugin +> is referenced directly when a logical replication slot is created. + +### 2. Create a logical replication slot using `wal2json` + +Once the cluster is ready, connect to the database with `psql` and create a +logical replication slot that uses the `wal2json` output plugin: + +```sql +SELECT * FROM pg_create_logical_replication_slot('wal2json_slot', 'wal2json'); +``` + +Some applications (for example, Teleport), may do this automatically. + +### 3. Verify installation + +Consume changes from the slot to confirm the plugin is loaded correctly. After +producing some activity (e.g. `CREATE TABLE t (id int); INSERT INTO t VALUES +(1);`), peek at the slot: + +```sql +SELECT data +FROM pg_logical_slot_peek_changes('wal2json_slot', NULL, NULL); +``` + +You should see the changes emitted as JSON documents. + +When you are done, drop the slot to release resources: + +```sql +SELECT pg_drop_replication_slot('wal2json_slot'); +``` + +## Contributors + +This extension is maintained by: + +- FirstName LastName (@GitHub_Handle) + +The maintainers are responsible for: + +- Monitoring upstream releases and security vulnerabilities. +- Ensuring compatibility with supported PostgreSQL versions. +- Reviewing and merging contributions specific to this extension's container + image and lifecycle. + +--- + +## Licenses and Copyright + +`wal2json`: + +- **Copyright:** (c) 2013-2018 Euler Taveira de Oliveira +- **License:** BSD 3-Clause License + +All relevant license and copyright information for the `wal2json` extension +and its dependencies are bundled within the image at: + +```text +/licenses/ +``` + +By using this image, you agree to comply with the terms of the licenses +contained therein. diff --git a/wal2json/metadata.hcl b/wal2json/metadata.hcl new file mode 100644 index 0000000..3675cd1 --- /dev/null +++ b/wal2json/metadata.hcl @@ -0,0 +1,33 @@ +# SPDX-FileCopyrightText: Copyright © contributors to CloudNativePG, established as CloudNativePG a Series of LF Projects, LLC. +# SPDX-License-Identifier: Apache-2.0 +metadata = { + name = "wal2json" + sql_name = "wal2json" + image_name = "wal2json" + licenses = ["BSD-3-Clause"] + shared_preload_libraries = [] + postgresql_parameters = {} + extension_control_path = [] + dynamic_library_path = [] + ld_library_path = [] + bin_path = [] + env = {} + auto_update_os_libs = false + required_extensions = [] + create_extension = false + + versions = { + bookworm = { + "18" = { + // renovate: suite=bookworm-pgdg depName=postgresql-18-wal2json + package = "2.6-3.pgdg12+1" + } + } + trixie = { + "18" = { + // renovate: suite=trixie-pgdg depName=postgresql-18-wal2json + package = "2.6-3.pgdg13+1" + } + } + } +} diff --git a/wal2json/test/chainsaw-test.yaml b/wal2json/test/chainsaw-test.yaml new file mode 100644 index 0000000..69c8d09 --- /dev/null +++ b/wal2json/test/chainsaw-test.yaml @@ -0,0 +1,36 @@ +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + name: verify-wal2json-output-plugin +spec: + timeouts: + apply: 5s + assert: 3m + delete: 30s + description: >- + Verify that the wal2json shared library is loadable by PostgreSQL by + creating and dropping a logical replication slot using the wal2json output + plugin. Slot creation forces PostgreSQL to dlopen wal2json.so, so this is + the actual end-to-end install check for an output plugin (which has no + CREATE EXTENSION counterpart). + steps: + - name: Provision PostgreSQL Cluster + description: >- + Deploy a Cluster resource using the built wal2json image with superuser + access enabled so the slot-creation Job can call + pg_create_logical_replication_slot. + try: + - apply: + file: cluster.yaml + - assert: + file: cluster-assert.yaml + + - name: Create logical replication slot using wal2json + description: >- + Run a Job that creates and drops a logical replication slot with the + 'wal2json' output plugin, exercising dlopen of wal2json.so. + try: + - apply: + file: check-slot.yaml + - assert: + file: check-slot-assert.yaml diff --git a/wal2json/test/check-slot-assert.yaml b/wal2json/test/check-slot-assert.yaml new file mode 100644 index 0000000..1230e21 --- /dev/null +++ b/wal2json/test/check-slot-assert.yaml @@ -0,0 +1,6 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: wal2json-slot-test +status: + succeeded: 1 diff --git a/wal2json/test/check-slot.yaml b/wal2json/test/check-slot.yaml new file mode 100644 index 0000000..794bf80 --- /dev/null +++ b/wal2json/test/check-slot.yaml @@ -0,0 +1,28 @@ +apiVersion: batch/v1 +kind: Job +metadata: + name: wal2json-slot-test +spec: + template: + spec: + restartPolicy: OnFailure + containers: + - name: slot-test + env: + - name: SU_URI + valueFrom: + secretKeyRef: + name: (join('-', [$values.name, 'superuser'])) + key: uri + image: alpine/psql:latest + command: ['sh', '-c'] + args: + - | + set -e + SU_URI=$(echo $SU_URI | sed "s|/\*|/|") + SLOT="wal2json_chainsaw_test" + psql "$SU_URI" -v ON_ERROR_STOP=1 -tAc \ + "SELECT slot_name FROM pg_create_logical_replication_slot('${SLOT}', 'wal2json')" + psql "$SU_URI" -v ON_ERROR_STOP=1 -tAc \ + "SELECT pg_drop_replication_slot('${SLOT}')" + echo "wal2json output plugin loaded successfully" diff --git a/wal2json/test/cluster-assert.yaml b/wal2json/test/cluster-assert.yaml new file mode 100644 index 0000000..aa22123 --- /dev/null +++ b/wal2json/test/cluster-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ($values.name) +status: + readyInstances: 1 + phase: Cluster in healthy state + image: ($values.pg_image) diff --git a/wal2json/test/cluster.yaml b/wal2json/test/cluster.yaml new file mode 100644 index 0000000..434953f --- /dev/null +++ b/wal2json/test/cluster.yaml @@ -0,0 +1,16 @@ +apiVersion: postgresql.cnpg.io/v1 +kind: Cluster +metadata: + name: ($values.name) +spec: + imageName: ($values.pg_image) + instances: 1 + enableSuperuserAccess: true + + storage: + size: 1Gi + + postgresql: + parameters: ($values.postgresql_parameters) + shared_preload_libraries: ($values.shared_preload_libraries) + extensions: ($values.extensions) From 9a8a9690043fac5e447fac2abb162e847d7a41cd Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Fri, 22 May 2026 18:29:21 +0000 Subject: [PATCH 2/7] chore: set `wal2json` maintainer info Set `wal2json` maintainer info to solidDoWant Signed-off-by: solidDoWant --- wal2json/README.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/wal2json/README.md b/wal2json/README.md index 5f3fa63..84f54aa 100644 --- a/wal2json/README.md +++ b/wal2json/README.md @@ -62,7 +62,8 @@ logical replication slot that uses the `wal2json` output plugin: SELECT * FROM pg_create_logical_replication_slot('wal2json_slot', 'wal2json'); ``` -Some applications (for example, Teleport), may do this automatically. +Some applications (for example, [Teleport](https://goteleport.com/docs/reference/deployment/backends/#postgresql)), +may do this automatically. ### 3. Verify installation @@ -87,7 +88,7 @@ SELECT pg_drop_replication_slot('wal2json_slot'); This extension is maintained by: -- FirstName LastName (@GitHub_Handle) +- Fred Heinecke (@solidDoWant) The maintainers are responsible for: From 65f2311546f0bb03fb714b4e87c95c21cf157b7f Mon Sep 17 00:00:00 2001 From: Gabriele Bartolini Date: Sat, 23 May 2026 10:01:42 +1000 Subject: [PATCH 3/7] chore: add CODEOWNER Signed-off-by: Gabriele Bartolini --- CODEOWNERS | 1 + 1 file changed, 1 insertion(+) diff --git a/CODEOWNERS b/CODEOWNERS index f4dd1e7..039b327 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,3 +3,4 @@ # https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/about-code-owners * @cloudnative-pg/maintainers @NiccoloFei +wal2json @solidDoWant From bfb50ef09d7cd0927be477f1a2ae33c249247f0c Mon Sep 17 00:00:00 2001 From: Gabriele Bartolini Date: Sat, 23 May 2026 10:04:49 +1000 Subject: [PATCH 4/7] chore: add wal2json to the main README Signed-off-by: Gabriele Bartolini --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0b9b4ed..898e2e3 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ they are maintained by their respective authors, and PostgreSQL Debian Group | **[pg_crash](pg-crash)** | **Disruptive** fault injection and chaos engineering extension | [github.com/cybertec-postgresql/pg_crash](https://github.com/cybertec-postgresql/pg_crash) | | **[pgvector](pgvector)** | Vector similarity search for PostgreSQL | [github.com/pgvector/pgvector](https://github.com/pgvector/pgvector) | | **[PostGIS](postgis)** | Geospatial database extension for PostgreSQL | [postgis.net/](https://postgis.net/) | - +| **[wal2json](wal2json)** | Logical decoding output plugin for PostgreSQL | [github.com/eulerto/wal2json](https://github.com/eulerto/wal2json) | Extensions are provided only for the OS versions already built by the [`cloudnative-pg/postgres-containers`](https://github.com/cloudnative-pg/postgres-containers) project, From 3ac9348e6275bdc4ee55dcfa4c4ce744ecd39f5f Mon Sep 17 00:00:00 2001 From: Gabriele Bartolini Date: Sat, 23 May 2026 10:07:30 +1000 Subject: [PATCH 5/7] docs: cosmetic change Signed-off-by: Gabriele Bartolini --- wal2json/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/wal2json/README.md b/wal2json/README.md index 84f54aa..f3db26f 100644 --- a/wal2json/README.md +++ b/wal2json/README.md @@ -68,8 +68,8 @@ may do this automatically. ### 3. Verify installation Consume changes from the slot to confirm the plugin is loaded correctly. After -producing some activity (e.g. `CREATE TABLE t (id int); INSERT INTO t VALUES -(1);`), peek at the slot: +producing some activity (e.g. `CREATE TABLE t (id int); INSERT INTO t VALUES (1);`), +peek at the slot: ```sql SELECT data From ed4f736b92d0997efd867b0e127bee1646ca9f5f Mon Sep 17 00:00:00 2001 From: Gabriele Bartolini Date: Sun, 24 May 2026 09:34:48 +1000 Subject: [PATCH 6/7] chore: fix CODEOWNERS Signed-off-by: Gabriele Bartolini --- CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CODEOWNERS b/CODEOWNERS index 039b327..f494385 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -3,4 +3,4 @@ # https://docs.github.com/en/free-pro-team@latest/github/creating-cloning-and-archiving-repositories/about-code-owners * @cloudnative-pg/maintainers @NiccoloFei -wal2json @solidDoWant +/wal2json/ @cloudnative-pg/maintainers @NiccoloFei @solidDoWant From ed4448b79643bc9ef9635634b99b91ebc44d1d56 Mon Sep 17 00:00:00 2001 From: solidDoWant Date: Wed, 27 May 2026 16:59:04 +0000 Subject: [PATCH 7/7] chore: switch `wal2json` tests to replication user instead of superuser Signed-off-by: solidDoWant --- wal2json/test/chainsaw-test.yaml | 2 +- wal2json/test/check-slot.yaml | 2 +- wal2json/test/cluster.yaml | 7 ++++++- 3 files changed, 8 insertions(+), 3 deletions(-) diff --git a/wal2json/test/chainsaw-test.yaml b/wal2json/test/chainsaw-test.yaml index 69c8d09..729bbea 100644 --- a/wal2json/test/chainsaw-test.yaml +++ b/wal2json/test/chainsaw-test.yaml @@ -16,7 +16,7 @@ spec: steps: - name: Provision PostgreSQL Cluster description: >- - Deploy a Cluster resource using the built wal2json image with superuser + Deploy a Cluster resource using the built wal2json image with replication access enabled so the slot-creation Job can call pg_create_logical_replication_slot. try: diff --git a/wal2json/test/check-slot.yaml b/wal2json/test/check-slot.yaml index 794bf80..f9ff195 100644 --- a/wal2json/test/check-slot.yaml +++ b/wal2json/test/check-slot.yaml @@ -12,7 +12,7 @@ spec: - name: SU_URI valueFrom: secretKeyRef: - name: (join('-', [$values.name, 'superuser'])) + name: (join('-', [$values.name, 'app'])) key: uri image: alpine/psql:latest command: ['sh', '-c'] diff --git a/wal2json/test/cluster.yaml b/wal2json/test/cluster.yaml index 434953f..d0ff391 100644 --- a/wal2json/test/cluster.yaml +++ b/wal2json/test/cluster.yaml @@ -5,7 +5,12 @@ metadata: spec: imageName: ($values.pg_image) instances: 1 - enableSuperuserAccess: true + managed: + roles: + - name: app + ensure: present + login: true + replication: true storage: size: 1Gi