pyln-testing: add SystemPostgresDbProvider for external postgres

.github/workflows/ci.yaml

@@ -356,6 +356,20 @@ jobs:
     runs-on: ubuntu-24.04
     needs:
       - check-compiled-source
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       fail-fast: false
       matrix:
@@ -365,7 +379,7 @@ jobs:
             TEST_NETWORK: regtest
             VALGRIND: 1
           - CFG: compile-gcc
-            TEST_DB_PROVIDER: postgres
+            TEST_DB_PROVIDER: postgres-system
             TEST_NETWORK: regtest
           - CFG: compile-gcc
             TEST_DB_PROVIDER: sqlite3
@@ -425,6 +439,7 @@ jobs:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
           TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
+          TEST_DB_PROVIDER_DSN: postgres://postgres:postgres@localhost:5432/postgres
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
           LIGHTNINGD_POSTGRES_NO_VACUUM: 1
           VALGRIND: ${{ matrix.VALGRIND }}
@@ -528,6 +543,20 @@ jobs:
       RUST_PROFILE: small # Has to match the one in the compile step
     needs:
       - first-integration
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER: postgres
+          POSTGRES_PASSWORD: postgres
+          POSTGRES_DB: postgres
+        ports:
+          - 5432:5432
+        options: >-
+          --health-cmd pg_isready
+          --health-interval 10s
+          --health-timeout 5s
+          --health-retries 5
     strategy:
       fail-fast: false
       matrix:
@@ -537,7 +566,7 @@ jobs:
           - NAME: postgres
             CFG: compile-gcc-O3
             COMPILER: gcc
-            TEST_DB_PROVIDER: postgres
+            TEST_DB_PROVIDER: postgres-system
             TEST_NETWORK: regtest
           # And don't forget about elements (like cdecker did when
           # reworking the CI...)
@@ -611,6 +640,7 @@ jobs:
           SLOW_MACHINE: 1
           TEST_DEBUG: 1
           TEST_DB_PROVIDER: ${{ matrix.TEST_DB_PROVIDER }}
+          TEST_DB_PROVIDER_DSN: postgres://postgres:postgres@localhost:5432/postgres
           TEST_NETWORK: ${{ matrix.TEST_NETWORK }}
           LIGHTNINGD_POSTGRES_NO_VACUUM: 1
           PYTEST_OPTS: ${{ env.PYTEST_OPTS_BASE }}

contrib/pyln-testing/pyln/testing/db.py

@@ -1,4 +1,5 @@
 from .utils import reserve_unused_port, drop_unused_port
+from urllib.parse import urlparse, urlunparse
 
 import itertools
 import logging
@@ -14,6 +15,18 @@
 from typing import Dict, List, Optional, Union
 
 
+def replace_dsn_database(dsn: str, dbname: str) -> str:
+    """Replace the database name in a PostgreSQL DSN.
+
+    Takes a DSN like 'postgres://user:pass@host:port/olddb' and returns
+    'postgres://user:pass@host:port/newdb'.
+    """
+    parsed = urlparse(dsn)
+    # Replace path (database name) with the new one
+    new_parsed = parsed._replace(path=f"/{dbname}")
+    return urlunparse(new_parsed)
+
+
 class BaseDb(object):
     def wipe_db(self):
         raise NotImplementedError("wipe_db method must be implemented by the subclass")
@@ -72,19 +85,26 @@ def wipe_db(self):
 
 
 class PostgresDb(BaseDb):
-    def __init__(self, dbname, port):
+    def __init__(self, dbname, port, base_dsn=None):
         self.dbname = dbname
         self.port = port
+        self.base_dsn = base_dsn
         self.provider = None
 
-        self.conn = psycopg2.connect("dbname={dbname} user=postgres host=localhost port={port}".format(
-            dbname=dbname, port=port
-        ))
+        if base_dsn:
+            # Connect using base DSN but with our specific database
+            self.conn = psycopg2.connect(replace_dsn_database(base_dsn, dbname))
+        else:
+            self.conn = psycopg2.connect("dbname={dbname} user=postgres host=localhost port={port}".format(
+                dbname=dbname, port=port
+            ))
         cur = self.conn.cursor()
         cur.execute('SELECT 1')
         cur.close()
 
     def get_dsn(self):
+        if self.base_dsn:
+            return replace_dsn_database(self.base_dsn, self.dbname)
         return "postgres://postgres:password@localhost:{port}/{dbname}".format(
             port=self.port, dbname=self.dbname
         )
@@ -118,10 +138,15 @@ def stop(self):
         """Clean up the database.
         """
         self.conn.close()
-        conn = psycopg2.connect("dbname=postgres user=postgres host=localhost port={self.port}")
+        if self.base_dsn:
+            conn = psycopg2.connect(self.base_dsn)
+        else:
+            conn = psycopg2.connect(f"dbname=postgres user=postgres host=localhost port={self.port}")
+        conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
         cur = conn.cursor()
         cur.execute("DROP DATABASE {};".format(self.dbname))
         cur.close()
+        conn.close()
 
     def wipe_db(self):
         cur = self.conn.cursor()
@@ -251,3 +276,59 @@ def stop(self):
         self.proc.wait()
         shutil.rmtree(self.pgdir)
         drop_unused_port(self.port)
+
+
+class SystemPostgresDbProvider(object):
+    """Use an existing system-wide PostgreSQL instance instead of spawning one.
+
+    This provider connects to an existing PostgreSQL server using a DSN from
+    the TEST_DB_PROVIDER_DSN environment variable. The DSN should point to a
+    database where the user has CREATE DATABASE privileges (typically the
+    'postgres' database with a superuser).
+
+    Example DSN: postgres://postgres:password@localhost:5432/postgres
+    """
+
+    def __init__(self, directory):
+        self.directory = directory
+        self.dsn = os.environ.get('TEST_DB_PROVIDER_DSN')
+        if not self.dsn:
+            raise ValueError(
+                "SystemPostgresDbProvider requires TEST_DB_PROVIDER_DSN environment variable"
+            )
+        self.conn = None
+        self.request = None  # Set by fixture to allow registering finalizers
+
+    def start(self):
+        self.conn = psycopg2.connect(self.dsn)
+        self.conn.set_isolation_level(psycopg2.extensions.ISOLATION_LEVEL_AUTOCOMMIT)
+        logging.info(f"Connected to system PostgreSQL via {self.dsn}")
+
+    def _drop_database(self, dbname):
+        """Drop a database, used as finalizer."""
+        try:
+            cur = self.conn.cursor()
+            cur.execute("DROP DATABASE IF EXISTS {};".format(dbname))
+            cur.close()
+        except Exception as e:
+            logging.warning(f"Failed to drop database {dbname}: {e}")
+
+    def get_db(self, node_directory, testname, node_id):
+        # Random suffix to avoid collisions on repeated tests
+        nonce = ''.join(random.choice(string.ascii_lowercase + string.digits) for _ in range(8))
+        dbname = "{}_{}_{}".format(testname, node_id, nonce)
+
+        cur = self.conn.cursor()
+        cur.execute("CREATE DATABASE {};".format(dbname))
+        cur.close()
+
+        # Register finalizer to drop this database at end of test
+        if self.request is not None:
+            self.request.addfinalizer(lambda db=dbname: self._drop_database(db))
+
+        db = PostgresDb(dbname, port=None, base_dsn=self.dsn)
+        return db
+
+    def stop(self):
+        if self.conn:
+            self.conn.close()

contrib/pyln-testing/pyln/testing/fixtures.py

@@ -1,5 +1,5 @@
 from concurrent import futures
-from pyln.testing.db import SqliteDbProvider, PostgresDbProvider
+from pyln.testing.db import SqliteDbProvider, PostgresDbProvider, SystemPostgresDbProvider
 from pyln.testing.utils import NodeFactory, BitcoinD, ElementsD, env, LightningNode, TEST_DEBUG, TEST_NETWORK, SLOW_MACHINE, VALGRIND
 from pyln.client import Millisatoshi
 from typing import Dict
@@ -718,12 +718,14 @@ def checkMemleak(node):
 providers = {
     'sqlite3': SqliteDbProvider,
     'postgres': PostgresDbProvider,
+    'postgres-system': SystemPostgresDbProvider,
 }
 
 
 @pytest.fixture
-def db_provider(test_base_dir):
+def db_provider(test_base_dir, request):
     provider = providers[os.getenv('TEST_DB_PROVIDER', 'sqlite3')](test_base_dir)
+    provider.request = request  # Allow provider to register finalizers
     provider.start()
     yield provider
     provider.stop()

pyproject.toml

@@ -20,6 +20,7 @@ dependencies = [
   "pyln-testing",
   "pyln-proto",
   "mnemonic>=0.21",
+  "flaky>=3.8.1",
 ]
 package-mode = false
 [dependency-groups]