From 5f6a7dbf21a20bfaa7fc06507d3d4e40edcdfcb6 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Tue, 2 Jun 2026 11:41:33 +0000 Subject: [PATCH 1/2] Move backup writer initialization to Coriolis core The Coriolis providers currently duplicate the code that initializes the backup writers and returns the backup writer connection info. We'll move that along with the corresponding constants to Coriolis core. Coriolis providers are free to define custom backup providers if necessary. --- coriolis/providers/backup_writers.py | 61 +++++++++++++++++++ .../tests/providers/test_backup_writers.py | 43 +++++++++++++ 2 files changed, 104 insertions(+) diff --git a/coriolis/providers/backup_writers.py b/coriolis/providers/backup_writers.py index 357a2f9d..4cc9a41a 100644 --- a/coriolis/providers/backup_writers.py +++ b/coriolis/providers/backup_writers.py @@ -22,6 +22,7 @@ from coriolis import constants from coriolis import data_transfer +from coriolis import events from coriolis import exception from coriolis.providers import provider_utils from coriolis import utils @@ -51,6 +52,18 @@ BACKUP_WRITER_FILE ] +# Common data transfer mechanisms exposed to Coriolis users. +DATA_TRANSFER_MECHANISM_SSH = "SSH" +DATA_TRANSFER_MECHANISM_HTTPS = "HTTPS" +# Can't be the same as replicator port. +DATA_TRANSFER_MECHANISM_HTTPS_PORT = 5566 + +# The file writer is meant for testing purposes and will not be exposed here. +DATA_TRANSFER_MECHANISM_MAP = { + DATA_TRANSFER_MECHANISM_SSH: BACKUP_WRITER_SSH, + DATA_TRANSFER_MECHANISM_HTTPS: BACKUP_WRITER_HTTP, +} + _WRITER_ERR_MAP = { -1: "ERR_MORE_MSG", 0: "ERR_DONE", @@ -176,6 +189,54 @@ def _validate_info(self, info): raise exception.CoriolisException( "Missing credentials in connection info") + @classmethod + def get_backup_writer_connection_info( + cls, + event_manager: events.EventManager, + ssh_connection_info: dict, + data_transfer_mechanism: str, + ) -> dict: + """Initialize the backup writer and obtain connection info. + + :param ssh_connection_info: a dict containing the following keys: + * ip + * port - usually SSH port (22) + * username + * password + * pkey - Paramiko keypair + :param data_transfer_mechanism: SSH or HTTPS + :returns: a dict describing the backend type and backup writer + connection details, used to subsequently retrieve the + backup writer. + """ + if data_transfer_mechanism == DATA_TRANSFER_MECHANISM_HTTPS: + event_manager.progress_update( + "Setting up HTTPS backup writer service on disk copy worker VM" + ) + writer_bootstrapper = HTTPBackupWriterBootstrapper( + ssh_connection_info, + DATA_TRANSFER_MECHANISM_HTTPS_PORT, + ) + https_conn_info = writer_bootstrapper.setup_writer() + writer_conn_info = { + "backend": DATA_TRANSFER_MECHANISM_MAP[ + DATA_TRANSFER_MECHANISM_HTTPS], + "connection_details": https_conn_info, + } + elif data_transfer_mechanism == DATA_TRANSFER_MECHANISM_SSH: + writer_conn_info = { + "backend": DATA_TRANSFER_MECHANISM_MAP[ + DATA_TRANSFER_MECHANISM_SSH], + "connection_details": ssh_connection_info, + } + else: + raise ValueError( + "Unhandleable data transfer mechanism '%s'" % ( + data_transfer_mechanism) + ) + + return writer_conn_info + class BaseBackupWriterImpl(with_metaclass(abc.ABCMeta)): def __init__(self, path, disk_id): diff --git a/coriolis/tests/providers/test_backup_writers.py b/coriolis/tests/providers/test_backup_writers.py index 4492336a..d0ab766f 100644 --- a/coriolis/tests/providers/test_backup_writers.py +++ b/coriolis/tests/providers/test_backup_writers.py @@ -164,6 +164,49 @@ def test__validate_info_with_missing_connection_details(self): self.assertRaises(exception.CoriolisException, self._get_factory, {"backend": "ssh"}) + @mock.patch.object(backup_writers, "HTTPBackupWriterBootstrapper") + def test_get_conn_info_https(self, mock_https_bootstrapper): + mechanism = backup_writers.DATA_TRANSFER_MECHANISM_HTTPS + factory = backup_writers.BackupWritersFactory + writer_conn_info = factory.get_backup_writer_connection_info( + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism=mechanism, + ) + self.assertEqual( + writer_conn_info["backend"], backup_writers.BACKUP_WRITER_HTTP) + self.assertEqual( + writer_conn_info["connection_details"], + mock_https_bootstrapper.return_value.setup_writer.return_value) + mock_https_bootstrapper.assert_called_once_with( + mock.sentinel.instance_conn_info, + backup_writers.DATA_TRANSFER_MECHANISM_HTTPS_PORT, + ) + + def test_get_conn_info_ssh(self): + mechanism = backup_writers.DATA_TRANSFER_MECHANISM_SSH + factory = backup_writers.BackupWritersFactory + writer_conn_info = factory.get_backup_writer_connection_info( + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism=mechanism, + ) + self.assertEqual( + writer_conn_info["backend"], backup_writers.BACKUP_WRITER_SSH) + self.assertEqual( + writer_conn_info["connection_details"], + mock.sentinel.instance_conn_info) + + def test_get_conn_info_unsupported(self): + factory = backup_writers.BackupWritersFactory + self.assertRaises( + ValueError, + factory.get_backup_writer_connection_info, + event_manager=mock.Mock(), + ssh_connection_info=mock.sentinel.instance_conn_info, + data_transfer_mechanism="fake-mechanism", + ) + class BaseBackupWriterTestCase(test_base.CoriolisBaseTestCase): """Test suite for the Coriolis BaseBackupWriter class.""" From 66574824b329315f11ea3f1bee9c789b21f6cab4 Mon Sep 17 00:00:00 2001 From: Lucian Petrut Date: Tue, 2 Jun 2026 13:52:15 +0000 Subject: [PATCH 2/2] Move the instance conn poller to Coriolis core Most providers use a copy of those helpers, which poll until the instance can be accessed using SSH or WINRM. For this reason, we'll move them to the Coriolis core utils. --- coriolis/constants.py | 4 ++ coriolis/utils.py | 94 +++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+) diff --git a/coriolis/constants.py b/coriolis/constants.py index 5624f130..0098131c 100644 --- a/coriolis/constants.py +++ b/coriolis/constants.py @@ -274,6 +274,10 @@ VALID_OS_TYPES = [OS_TYPE_BSD, OS_TYPE_LINUX, OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS] +PROTOCOL_SSH = "ssh" +# WinRM is technically the Microsoft implementation of WSMAN. +PROTOCOL_WINRM = "winrm" + TMP_DIRS_KEY = "__tmp_dirs" COMPRESSION_FORMAT_GZIP = "gzip" diff --git a/coriolis/utils.py b/coriolis/utils.py index 08d50171..2266f85a 100644 --- a/coriolis/utils.py +++ b/coriolis/utils.py @@ -1118,3 +1118,97 @@ def start_thread(target, args=(), kwargs=None, daemon=True): ) thread.start() return thread + + +def _poll_instance_until_reachable_ssh( + connection_info: dict, + timeout: int = 600, + poll_interval: int = 10, +): + start = time.time() + while (time.time() - start) < timeout: + try: + ssh = connect_ssh( + hostname=connection_info["ip"], + port=connection_info["port"], + username=connection_info["username"], + password=connection_info["password"], + pkey=connection_info["pkey"], + ) + # "exit 0" should work across platforms. "whoami" would also work. + exec_ssh_cmd(ssh, "exit 0") + ssh.close() + LOG.debug("Instance reachable: %s", connection_info["ip"]) + return + except Exception as err: + LOG.debug( + f"Could not connect to remote instance: {str(err)}. " + f"Retrying, time left: {timeout - (time.time() - start)}." + ) + time.sleep(poll_interval) + + raise exception.CoriolisException( + f"Operation timed out after waiting {timeout}s for the instance to be " + f"accessible via SSH." + ) + + +def _poll_instance_until_reachable_winrm( + connection_info: dict, + timeout: int = 600, + poll_interval: int = 10, +): + start = time.time() + while (time.time() - start) < timeout: + try: + # Avoid a circular dependency. + from coriolis import wsman + conn = wsman.WSManConnection.from_connection_info(connection_info) + conn.exec_ps_command("whoami") + return + except Exception as ex: + LOG.debug( + f"Could not conect to Windows host: {str(ex)}. " + f"Retrying, time left: {timeout - (time.time() - start)}." + ) + time.sleep(poll_interval) + + raise exception.CoriolisException( + f"Operation timed out after waiting {timeout}s for Windows host to " + f"be accessible via WinRM." + ) + + +def poll_instance_until_reachable( + connection_info: dict, + protocol: str = constants.PROTOCOL_SSH, + timeout: int = 600, + poll_interval: int = 10, +) -> paramiko.SSHClient: + """Poll until a given instance becomes reachable. + + :param connection_info: a dict containing the following keys: + * ip + * port + * username + * password + * pkey - Paramiko keypair + :param protocol: connection protocol, "ssh" or "winrm" + :param timeout: the maximum amount of time to wait + :param poll_interval: the amount of time to wait between retries + """ + # TODO(lpetrut): consider including the connection protocol in the + # connection info. We'd have to modify a few schemas used during os + # morphing. We currently pick the protocol based on the OS type but + # we may want to use SSH on Windows as well. + if protocol == constants.PROTOCOL_SSH: + helper = _poll_instance_until_reachable_ssh + elif protocol == constants.PROTOCOL_WINRM: + helper = _poll_instance_until_reachable_winrm + else: + raise exception.InvalidInput( + f"Unsupported instance connection protocol: {protocol}") + return helper( + connection_info=connection_info, + timeout=timeout, + poll_interval=poll_interval)