Move common utils to Coriolis core

coriolis/constants.py

@@ -274,6 +274,10 @@
 VALID_OS_TYPES = [OS_TYPE_BSD, OS_TYPE_LINUX,
                   OS_TYPE_OS_X, OS_TYPE_SOLARIS, OS_TYPE_WINDOWS]
 
+PROTOCOL_SSH = "ssh"
+# WinRM is technically the Microsoft implementation of WSMAN.
+PROTOCOL_WINRM = "winrm"
+
 TMP_DIRS_KEY = "__tmp_dirs"
 
 COMPRESSION_FORMAT_GZIP = "gzip"

coriolis/providers/backup_writers.py

@@ -22,6 +22,7 @@
 
 from coriolis import constants
 from coriolis import data_transfer
+from coriolis import events
 from coriolis import exception
 from coriolis.providers import provider_utils
 from coriolis import utils
@@ -51,6 +52,18 @@
     BACKUP_WRITER_FILE
 ]
 
+# Common data transfer mechanisms exposed to Coriolis users.
+DATA_TRANSFER_MECHANISM_SSH = "SSH"
+DATA_TRANSFER_MECHANISM_HTTPS = "HTTPS"
+# Can't be the same as replicator port.
+DATA_TRANSFER_MECHANISM_HTTPS_PORT = 5566
+
+# The file writer is meant for testing purposes and will not be exposed here.
+DATA_TRANSFER_MECHANISM_MAP = {
+    DATA_TRANSFER_MECHANISM_SSH: BACKUP_WRITER_SSH,
+    DATA_TRANSFER_MECHANISM_HTTPS: BACKUP_WRITER_HTTP,
+}
+
 _WRITER_ERR_MAP = {
     -1: "ERR_MORE_MSG",
     0: "ERR_DONE",
@@ -176,6 +189,54 @@ def _validate_info(self, info):
             raise exception.CoriolisException(
                 "Missing credentials in connection info")
 
+    @classmethod
+    def get_backup_writer_connection_info(
+        cls,
+        event_manager: events.EventManager,
+        ssh_connection_info: dict,
+        data_transfer_mechanism: str,
+    ) -> dict:
+        """Initialize the backup writer and obtain connection info.
+
+        :param ssh_connection_info: a dict containing the following keys:
+            * ip
+            * port - usually SSH port (22)
+            * username
+            * password
+            * pkey - Paramiko keypair
+        :param data_transfer_mechanism: SSH or HTTPS
+        :returns: a dict describing the backend type and backup writer
+                  connection details, used to subsequently retrieve the
+                  backup writer.
+        """
+        if data_transfer_mechanism == DATA_TRANSFER_MECHANISM_HTTPS:
+            event_manager.progress_update(
+                "Setting up HTTPS backup writer service on disk copy worker VM"
+            )
+            writer_bootstrapper = HTTPBackupWriterBootstrapper(
+                ssh_connection_info,
+                DATA_TRANSFER_MECHANISM_HTTPS_PORT,
+            )
+            https_conn_info = writer_bootstrapper.setup_writer()
+            writer_conn_info = {
+                "backend": DATA_TRANSFER_MECHANISM_MAP[
+                    DATA_TRANSFER_MECHANISM_HTTPS],
+                "connection_details": https_conn_info,
+            }
+        elif data_transfer_mechanism == DATA_TRANSFER_MECHANISM_SSH:
+            writer_conn_info = {
+                "backend": DATA_TRANSFER_MECHANISM_MAP[
+                    DATA_TRANSFER_MECHANISM_SSH],
+                "connection_details": ssh_connection_info,
+            }
+        else:
+            raise ValueError(
+                "Unhandleable data transfer mechanism '%s'" % (
+                    data_transfer_mechanism)
+            )
+
+        return writer_conn_info
+
 
 class BaseBackupWriterImpl(with_metaclass(abc.ABCMeta)):
     def __init__(self, path, disk_id):

coriolis/tests/providers/test_backup_writers.py

@@ -164,6 +164,49 @@ def test__validate_info_with_missing_connection_details(self):
         self.assertRaises(exception.CoriolisException,
                           self._get_factory, {"backend": "ssh"})
 
+    @mock.patch.object(backup_writers, "HTTPBackupWriterBootstrapper")
+    def test_get_conn_info_https(self, mock_https_bootstrapper):
+        mechanism = backup_writers.DATA_TRANSFER_MECHANISM_HTTPS
+        factory = backup_writers.BackupWritersFactory
+        writer_conn_info = factory.get_backup_writer_connection_info(
+            event_manager=mock.Mock(),
+            ssh_connection_info=mock.sentinel.instance_conn_info,
+            data_transfer_mechanism=mechanism,
+        )
+        self.assertEqual(
+            writer_conn_info["backend"], backup_writers.BACKUP_WRITER_HTTP)
+        self.assertEqual(
+            writer_conn_info["connection_details"],
+            mock_https_bootstrapper.return_value.setup_writer.return_value)
+        mock_https_bootstrapper.assert_called_once_with(
+            mock.sentinel.instance_conn_info,
+            backup_writers.DATA_TRANSFER_MECHANISM_HTTPS_PORT,
+        )
+
+    def test_get_conn_info_ssh(self):
+        mechanism = backup_writers.DATA_TRANSFER_MECHANISM_SSH
+        factory = backup_writers.BackupWritersFactory
+        writer_conn_info = factory.get_backup_writer_connection_info(
+            event_manager=mock.Mock(),
+            ssh_connection_info=mock.sentinel.instance_conn_info,
+            data_transfer_mechanism=mechanism,
+        )
+        self.assertEqual(
+            writer_conn_info["backend"], backup_writers.BACKUP_WRITER_SSH)
+        self.assertEqual(
+            writer_conn_info["connection_details"],
+            mock.sentinel.instance_conn_info)
+
+    def test_get_conn_info_unsupported(self):
+        factory = backup_writers.BackupWritersFactory
+        self.assertRaises(
+            ValueError,
+            factory.get_backup_writer_connection_info,
+            event_manager=mock.Mock(),
+            ssh_connection_info=mock.sentinel.instance_conn_info,
+            data_transfer_mechanism="fake-mechanism",
+        )
+
 
 class BaseBackupWriterTestCase(test_base.CoriolisBaseTestCase):
     """Test suite for the Coriolis BaseBackupWriter class."""

coriolis/utils.py

@@ -1118,3 +1118,97 @@ def start_thread(target, args=(), kwargs=None, daemon=True):
     )
     thread.start()
     return thread
+
+
+def _poll_instance_until_reachable_ssh(
+    connection_info: dict,
+    timeout: int = 600,
+    poll_interval: int = 10,
+):
+    start = time.time()
+    while (time.time() - start) < timeout:
+        try:
+            ssh = connect_ssh(
+                hostname=connection_info["ip"],
+                port=connection_info["port"],
+                username=connection_info["username"],
+                password=connection_info["password"],
+                pkey=connection_info["pkey"],
+            )
+            # "exit 0" should work across platforms. "whoami" would also work.
+            exec_ssh_cmd(ssh, "exit 0")
+            ssh.close()
+            LOG.debug("Instance reachable: %s", connection_info["ip"])
+            return
+        except Exception as err:
+            LOG.debug(
+                f"Could not connect to remote instance: {str(err)}. "
+                f"Retrying, time left: {timeout - (time.time() - start)}."
+            )
+        time.sleep(poll_interval)
+
+    raise exception.CoriolisException(
+        f"Operation timed out after waiting {timeout}s for the instance to be "
+        f"accessible via SSH."
+    )
+
+
+def _poll_instance_until_reachable_winrm(
+    connection_info: dict,
+    timeout: int = 600,
+    poll_interval: int = 10,
+):
+    start = time.time()
+    while (time.time() - start) < timeout:
+        try:
+            # Avoid a circular dependency.
+            from coriolis import wsman
+            conn = wsman.WSManConnection.from_connection_info(connection_info)
+            conn.exec_ps_command("whoami")
+            return
+        except Exception as ex:
+            LOG.debug(
+                f"Could not conect to Windows host: {str(ex)}. "
+                f"Retrying, time left: {timeout - (time.time() - start)}."
+            )
+        time.sleep(poll_interval)
+
+    raise exception.CoriolisException(
+        f"Operation timed out after waiting {timeout}s for Windows host to "
+        f"be accessible via WinRM."
+    )
+
+
+def poll_instance_until_reachable(
+    connection_info: dict,
+    protocol: str = constants.PROTOCOL_SSH,
+    timeout: int = 600,
+    poll_interval: int = 10,
+) -> paramiko.SSHClient:
+    """Poll until a given instance becomes reachable.
+
+    :param connection_info: a dict containing the following keys:
+        * ip
+        * port
+        * username
+        * password
+        * pkey - Paramiko keypair
+    :param protocol: connection protocol, "ssh" or "winrm"
+    :param timeout: the maximum amount of time to wait
+    :param poll_interval: the amount of time to wait between retries
+    """
+    # TODO(lpetrut): consider including the connection protocol in the
+    # connection info. We'd have to modify a few schemas used during os
+    # morphing. We currently pick the protocol based on the OS type but
+    # we may want to use SSH on Windows as well.
+    if protocol == constants.PROTOCOL_SSH:
+        helper = _poll_instance_until_reachable_ssh
+    elif protocol == constants.PROTOCOL_WINRM:
+        helper = _poll_instance_until_reachable_winrm
+    else:
+        raise exception.InvalidInput(
+            f"Unsupported instance connection protocol: {protocol}")
+    return helper(
+        connection_info=connection_info,
+        timeout=timeout,
+        poll_interval=poll_interval)