diff --git a/coriolis/constants.py b/coriolis/constants.py index 71e55343..ef1f795f 100644 --- a/coriolis/constants.py +++ b/coriolis/constants.py @@ -188,8 +188,6 @@ PROVIDER_PLATFORM_SOURCE = "source" PROVIDER_PLATFORM_DESTINATION = "destination" -PROVIDER_TYPE_IMPORT = 1 -PROVIDER_TYPE_EXPORT = 2 PROVIDER_TYPE_TRANSFER_IMPORT = 4 PROVIDER_TYPE_TRANSFER_EXPORT = 8 PROVIDER_TYPE_ENDPOINT = 16 @@ -211,6 +209,8 @@ # and de-increment the rest PROVIDER_TYPE_VALIDATE_MIGRATION_EXPORT = 2048 PROVIDER_TYPE_VALIDATE_MIGRATION_IMPORT = 8192 +PROVIDER_TYPE_IMPORT = 1 +PROVIDER_TYPE_EXPORT = 2 DISK_FORMAT_VMDK = 'vmdk' DISK_FORMAT_RAW = 'raw' diff --git a/coriolis/providers/base.py b/coriolis/providers/base.py index 8f39d221..dbfb8c58 100644 --- a/coriolis/providers/base.py +++ b/coriolis/providers/base.py @@ -7,7 +7,10 @@ from oslo_log import log as logging from six import with_metaclass +from coriolis import context from coriolis import exception +from coriolis.osmorphing import base as base_osmorphing +from coriolis.osmorphing.osdetect import base as base_osdetect LOG = logging.getLogger(__name__) @@ -15,18 +18,32 @@ class BaseProvider(object, with_metaclass(abc.ABCMeta)): @property - def platform(self): + def platform(self) -> str: + """Platform type.""" raise NotImplementedError("Missing provider platform attribute.") class BaseEndpointProvider(BaseProvider): @abc.abstractmethod - def validate_connection(self, ctxt, connection_info): + def validate_connection( + self, + ctxt: context.RequestContext, + connection_info: dict, + ): + """Validate the endpoint connection info. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + + Schema validation is not performed by the caller and must + be handled by the provider. + """ pass @abc.abstractmethod - def get_connection_info_schema(self): + def get_connection_info_schema(self) -> str: + """Get the provider specific connection info schema.""" pass @@ -34,39 +51,90 @@ class BaseEndpointInstancesProvider(BaseEndpointProvider): """Defines operations for listing instances off of Endpoints.""" @abc.abstractmethod - def get_instances(self, ctxt, connection_info, source_environment, - limit=None, last_seen_id=None, - instance_name_pattern=None, refresh=False): + def get_instances( + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + limit: int | None = None, + last_seen_id: str | None = None, + instance_name_pattern: str | None = None, + refresh: bool = False, + ) -> list[dict]: """Returns a list of instances. - :param refresh: If True, forces a refresh of any cached instance data. + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param limit: the maximum number of instances to retrieve + :param last_seen_id: the last seen instance id, used for pagination + :param instance_name_pattern: a name pattern used for filtering + instances + :param refresh: if True, forces a refresh of any cached instance data + :returns: a list of dicts conforming to vm_instance_info_schema.json """ raise NotImplementedError() @abc.abstractmethod def get_instance( - self, ctxt, connection_info, source_environment, instance_name): - """Returns detailed info for a given instance.""" + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + instance_name: str, + ) -> dict: + """Returns detailed info for a given instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param instance_name: the name of the instance to retrieve. + :returns: a dict conforming to vm_instance_info_schema.json + + TODO: Identical to "get_replica_instance_info", one of them should be + deprecated. + """ raise NotImplementedError() class BaseEndpointNetworksProvider(object, with_metaclass(abc.ABCMeta)): - """Defines operations for endpoints networks.""" + """Defines operations for endpoint networks.""" @abc.abstractmethod - def get_networks(self, ctxt, connection_info, env): - """Returns a list of networks """ + def get_networks( + self, + ctxt: context.RequestContext, + connection_info: dict, + env: dict, + ) -> list[dict]: + """Returns a list of network identifiers. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param env: provider specific environment options + :returns: a list of dicts containing the "name" and "id" fields + """ raise NotImplementedError() class BaseProviderSetupExtraLibsMixin(object, with_metaclass(abc.ABCMeta)): - """ ABC mixin for providers which require extra libraries loaded. """ + """ABC mix-in for providers which require extra libraries loaded.""" @abc.abstractmethod - def get_shared_library_directories(self, ctxt, connection_info): - """ Should return a list of string paths to directories somewhere in - the worker filesystem where extra libraries required for the provider - are located. + def get_shared_library_directories( + self, + ctxt: context.RequestContext, + connection_info: dict, + ) -> list[str]: + """Get directories containing shared libraries needed by the provider. + + The specified directories are expected to reside on the worker + filesystem. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters """ return [] @@ -75,13 +143,20 @@ class BaseEndpointDestinationOptionsProvider( object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_target_environment_options( - self, ctxt, connection_info, env=None, option_names=None): - """ Returns all possible values for the target environment options, as + self, + ctxt: context.RequestContext, + connection_info: dict, + env: dict | None = None, + option_names: list[str] | None = None, + ) -> list[dict]: + """Returns all possible values for the target environment options, as well as any settings the options might have in the configuration files. - param env: dict: optional target environment options - param option_names: list(str): optional list of parameter names to show - values for + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param env: dict: optional target environment options + :param option_names: optional list of parameter names to + show values for Example returned values for the following options: schema = { @@ -122,7 +197,7 @@ def get_target_environment_options( Observations: - base types such as 'integer' or 'string' are preserved - 'array' types will return an array with all the options which are - settable through that paramter (any, all or none may be set) + settable through that parameter (any, all or none may be set) - for fields where both a name or ID may be returned, returning the name will be preferred. The provider must ensure that, if there are objects with the same name, the IDs of those objects are @@ -135,13 +210,20 @@ class BaseEndpointSourceOptionsProvider( object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod def get_source_environment_options( - self, ctxt, connection_info, env=None, option_names=None): - """ Returns all possible values for the source environment options, as + self, + ctxt: context.RequestContext, + connection_info: dict, + env: dict | None = None, + option_names: list[str] | None = None, + ) -> list[dict]: + """Returns all possible values for the source environment options, as well as any settings the options might have in the configuration files. - param env: dict: optional target environment options - param option_names: list(str): optional list of parameter names to show - values for + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param env: dict: optional target environment options + :param option_names: optional list of parameter names + to show values for Example returned values for the following options: schema = { @@ -182,7 +264,7 @@ def get_source_environment_options( Observations: - base types such as 'integer' or 'string' are preserved - 'array' types will return an array with all the options which are - settable through that paramter (any, all or none may be set) + settable through that parameter (any, all or none may be set) - for fields where both a name or ID may be returned, returning the name will be preferred. The provider must ensure that, if there are objects with the same name, the IDs of those objects are @@ -194,19 +276,38 @@ def get_source_environment_options( class BaseInstanceProvider(BaseProvider): @abc.abstractmethod - def get_os_morphing_tools(self, os_type, osmorphing_info): - """ Returns a list of possible OSMorphing classes for the given + def get_os_morphing_tools( + self, + os_type: str, + osmorphing_info: dict, + ) -> list[base_osmorphing.BaseOSMorphingTools]: + """Returns a list of possible OSMorphing classes for the given os type and osmorphing info. + + :param os_type: an operating system type such as "linux" or "windows". + See the list of constants for other OS types. + :param osmorphing_info: a set of parameters controlling the OS morphing + process. See os_morphing_resources_schema.json. + The OSMorphing classes will be asked to validate compatibility in order using their `check_os` method in order, so any classes whose - `check_os` classmethods might both return a positive result should be + `check_os` class methods might both return a positive result should be placed in the correct order (from more specific to less specific). """ raise exception.OSMorphingToolsNotFound(os_type=os_type) - def get_custom_os_detect_tools(self, os_type, osmorphing_info): - """ Returns a list of custom OSDetect classes which inherit from - coriolis.osmorphing.osdetect.base.BaseOSDetectTools. + def get_custom_os_detect_tools( + self, + os_type: str, + osmorphing_info: dict, + ) -> list[base_osdetect.BaseOSDetectTools]: + """Returns a list of custom OSDetect classes. + + :param os_type: an operating system type such as "linux" or "windows". + See the list of constants for other OS types. + :param osmorphing_info: a set of parameters controlling the OS morphing + process. See os_morphing_resources_schema.json. + These detect tools will be run before the standard ones already present in the standard coriolis.osmorphing.osdetect module in case there will be any provider-specific supported OS releases. @@ -217,10 +318,29 @@ def get_custom_os_detect_tools(self, os_type, osmorphing_info): class BaseImportInstanceProvider(BaseInstanceProvider): @abc.abstractmethod - def get_target_environment_schema(self): + def get_target_environment_schema(self) -> str: + """Retrieve the provider specific target environment schema. + + Users can specify target environment parameters to control the way + in which replicas and minions get deployed on the destination side by + the import provider. + + These settings are provided at run time and usually take precedence + over the coriolis-worker configuration. + """ pass - def _get_destination_instance_name(self, export_info, instance_name): + def _get_destination_instance_name( + self, + export_info: dict, + instance_name: str, + ) -> str: + """Helper to determine the preferred destination instance name. + + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param instance_name: fallback instance name + """ dest_instance_name = export_info.get("name", instance_name) LOG.debug('Destination instance name for "%(instance_name)s": ' '"%(dest_instance_name)s"', @@ -230,110 +350,131 @@ def _get_destination_instance_name(self, export_info, instance_name): @abc.abstractmethod def deploy_os_morphing_resources( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_deployment_info: dict, + ) -> dict: + """Deploy resources used as part of the OS morphing process. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: user provided target environment parameters + :param instance_deployment_info: target instance information returned + by the provider through the + "deploy_replica_instance" method + :returns: a dict containing the following fields: + * os_morphing_resources + * osmorphing_connection_info + * osmorphing_info + See os_morphing_resource_schema.json for the full schema. + + The provider will deploy a minion instance and attach the volumes + specified in the "instance_deployment_info". The returned connection + info will be used by the caller to initiate the OS morphing process, + leveraging the OS morphing manager. + + In case of failure, any created resources must be cleaned up by the + provider. + """ pass @abc.abstractmethod def delete_os_morphing_resources( - self, ctxt, connection_info, target_environment, - os_morphing_resources): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + os_morphing_resources: dict, + ): + """Cleanup resources used as part of the OS morphing process. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: user provided target environment parameters + :param os_morphing_resources: the dict returned by the provider + through "deploy_os_morphing_resources" + """ pass -class BaseMigrationExportValidationProvider( - object, with_metaclass(abc.ABCMeta)): - """ Defines methods to be called for migration export input validation """ - - @abc.abstractmethod - def validate_migration_export_input( - self, ctxt, connection_info, instance_name, source_environment): - """ Should verify the provided 'connection_info' and - 'source_environment' and return the expected Migration - export info for the given VM. """ - return {} - - class BaseReplicaExportValidationProvider( object, with_metaclass(abc.ABCMeta)): - """ Defines methods to be called for replica export input validation """ + """Validate replica export parameters.""" @abc.abstractmethod def validate_replica_export_input( - self, ctxt, connection_info, instance_name, source_environment): - """ Should verify the provided 'connection_info' and - 'source_environment' and return the expected Migration - export info for the given VM. """ + self, + ctxt: context.RequestContext, + connection_info: dict, + instance_name: str, + source_environment: dict, + ) -> dict: + """Validate the parameters and return the replica export info. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param instance_name: the name of the instance to export + :param source_environment: provider specific source environment + parameters + :returns: the export info for the given instance, + conforming to vm_export_info_schema.json + """ return {} -class BaseMigrationImportValidationProvider( - object, with_metaclass(abc.ABCMeta)): - """ Defines methods to be called for migration import input validation """ - - @abc.abstractmethod - def validate_migration_import_input( - self, ctxt, connection_info, target_environment, export_info): - """ Validates the provided Migration parameters """ - pass - - class BaseReplicaImportValidationProvider( object, with_metaclass(abc.ABCMeta)): - """ Defines methods to be called for replica import input validation """ + """Validate replica import parameters.""" @abc.abstractmethod def validate_replica_import_input( - self, ctxt, connection_info, target_environment, export_info, - check_os_morphing_resources=False, check_final_vm_params=False): - """ Validates the provided Replica parameters """ - pass - - @abc.abstractmethod - def validate_replica_deployment_input( - self, ctxt, connection_info, target_environment, export_info): - """ Validates the provided Replica deployment parameters """ - pass - - -class BaseImportProvider(BaseImportInstanceProvider): - - @abc.abstractmethod - def import_instance(self, ctxt, connection_info, target_environment, - instance_name, export_info): - """Imports the given instance. - - Imports the instance given by its name to the specified target - environment within the destination cloud based on the provided - connection and export info. + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + export_info: dict, + check_os_morphing_resources: bool = False, + check_final_vm_params: bool = False, + ): + """Validates the provided replica import parameters. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param check_os_morphing_resources: if set, the provider is expected to + validate OS morphing parameters + :param check_final_vm_params: if set, the provider is expected to + validate replica instance parameters """ pass @abc.abstractmethod - def deploy_disk_copy_resources( - self, ctxt, connection_info, target_environment, volumes_info): - pass - - @abc.abstractmethod - def delete_disk_copy_resources( - self, ctxt, connection_info, target_environment, - target_resources_dict): - pass - - @abc.abstractmethod - def finalize_import_instance( - self, ctxt, connection_info, target_environment, - instance_deployment_info): - """ Should return a dict with the info of the migrated VM on the - destination platform in the same format as offered by - 'BaseExportProvider.export_instance()'. + def validate_replica_deployment_input( + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + export_info: dict, + ): + """Validates the provided replica import parameters. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + + Same as "validate_replica_import_input", except that it doesn't + explicitly state which set of parameters to validate. + + TODO: consider deprecating one of those two. """ - return {} - - @abc.abstractmethod - def cleanup_failed_import_instance( - self, ctxt, connection_info, target_environment, - instance_deployment_info): pass @@ -341,80 +482,343 @@ class BaseReplicaImportProvider(BaseImportInstanceProvider): @abc.abstractmethod def deploy_replica_instance( - self, ctxt, connection_info, target_environment, - instance_name, export_info, volumes_info, clone_disks): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_name: str, + export_info: dict, + volumes_info: list[dict], + clone_disks: bool, + ) -> dict: + """Deploy replica instance resources. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param instance_name: the migrated instance name. Use the + "_get_destination_instance_name" to determine + the expected destination name. + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + :param clone_disks: whether the specified volumes should be cloned or + attached directly to the instance. + :returns: a provider specific dict defining instance characteristics + + At this point, the volumes specified in "volumes_info" are expected to + contain the transferred data. + + "finalize_replica_instance_deployment" will be called afterwards to + finalize the deployment. + """ pass @abc.abstractmethod def finalize_replica_instance_deployment( - self, ctxt, connection_info, target_environment, - instance_deployment_info): - """ Should return a dict with the info of the migrated VM on the - destination platform in the same format as offered by - 'BaseExportProvider.export_instance()'. + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_deployment_info: dict, + ) -> dict: + """Finalize the replica instance deployment. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param instance_deployment_info: instance info returned by the provider + through "deploy_replica_instance" + :returns: a dict describing the migrated instance on the destination + platform, expected to have the same format as the export + info declared in "vm_export_info_schema.json". """ return {} @abc.abstractmethod def cleanup_failed_replica_instance_deployment( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_deployment_info: dict, + ): + """Cleanup a failed replica instance deployment. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param instance_deployment_info: instance info returned by the provider + through "deploy_replica_instance" + """ pass @abc.abstractmethod def deploy_replica_disks( - self, ctxt, connection_info, target_environment, instance_name, - export_info, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_name: str, + export_info: dict, + volumes_info: list[dict], + ) -> list[dict]: + """Create or update the replica disks. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param instance_name: the migrated instance name. Use + "_get_destination_instance_name" to determine + the expected destination name. + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + It starts as an empty list and gets populated + by subsequent "deploy_replica_disks" provider + calls, performed as part of disk transfer + operations. + :returns: the updated list of volumes. + + The disks defined by the export info must be replicated on the + destination cloud. The provider will recreate the volumes based on the + size specified in the export info and other user provided + "target_environment" parameters. + + The resulting volume info is expected to contain the original disk + id from the export info, along with any other provider specific volume + properties. + + "deploy_replica_disks" will be called whenever the user initiates a + disk transfer. Be aware that the exported instance may be modified + between Coriolis transfers. + + As such, the provider is expected to compare the "volumes_info" + that it returned last time (now received as input parameter) with the + current export info. It should then determine which disks need to be + created, deleted or resized. + + Note that "deploy_replica_disks" is not expected to transfer data. That + will be handled separately. + + If multiple storage backends are available, the provider may report + them through the "get_storage" method. The "target_environment" + parameter will then contain "storage_mappings", associating source + storage backend identifiers with destination backend identifiers. The + export provider will use this information to place the volumes + (also called disks throughout the API) on the backend requested by + the user. + """ pass @abc.abstractmethod def deploy_replica_target_resources( - self, ctxt, connection_info, target_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + volumes_info: list[dict], + ) -> dict: + """Create a minion instance used for disk transfers. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + :returns: a dict containing the following fields: + * migr_resources: provider specific dict describing the minion + instance + * volumes_info: updated list of volumes, containing + attachment path. See the below explanation for + setting "volume_dev". + * connection_info: a dict containing minion connection details, + conforming to disk_sync_resources_conn_info_schema.json and + containing the following: + * backend: backup writer type, one of: + * "ssh_backup_writer" + * "http_backup_writer" + * "file_backup_writer" + * connection_details: + connection details returned by the writer, conforming to + disk_sync_resources_conn_info_schema.json. + See backup_writers.py. + + The provider will need to initialize the backup writer, passing it a + connection info dict that usually contains the minion ip, a port used + by the backup writer, credentials and/or a Paramiko keypair. + + The volumes contained by "volumes_info" must include the "volume_dev" + field, specifying the attachment path as seen by the VM. Disk paths + such as "/dev/sdb" must be handled carefully since the device naming + can change depending on the order in which the disks are identified by + the guest. + + Providers that can reliably determine the address or serial ID of the + disk are encouraged to use udev links such as: + * /dev/disk/by-path/pci-0000:18:00.0-scsi-0:2:0:0 + * /dev/disk/by-id/wwn-0x6d094660793802002afcbbe61cfbcd38 + * /dev/disk/by-id/scsi-36d094660793802002afcbbe61cfbcd38 + * /dev/disk/by-id/virtio-98fa455fc976 + + Note that some providers need to rely on device names and leverage the + replicator service to determine the paths identified by the guest. The + workflow consists in attaching one disk at a time and identifying the + last attached disk. + + Providers that support minion pools must still implement this method, + which will be called if the user chooses not to use a minion pool. + + In case of failure, the provider is expected to perform cleanup. + """ pass @abc.abstractmethod def delete_replica_target_resources( - self, ctxt, connection_info, target_environment, - migr_resources_dict): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + migr_resources_dict: dict, + ): + """Cleanup any resources created by "deploy_replica_target_resources". + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param migr_resources_dict: provider specific dict returned by + 'deploy_replica_target_resources" + """ pass @abc.abstractmethod def delete_replica_disks( - self, ctxt, connection_info, target_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + volumes_info: list[dict], + ): + """Delete replica disks created as part of the transfer process. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + """ pass @abc.abstractmethod def create_replica_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + volumes_info: list[dict], + ) -> list[dict]: + """Snapshot replica disks created as part of the transfer process. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + :returns: updated volume info, containing snapshot information. + + Snapshots are meant to speed up replica deployments. Before initiating + the deployment, Coriolis will snapshot the replica disks, which will + then be attached to the replica instance. + + The replica disks will be recreated based on the snapshots, to be used + by subsequent disk transfers or replica deployments. + + If snapshots are unsupported, volume cloning will be used instead. + """ pass @abc.abstractmethod def delete_replica_target_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + volumes_info: list[dict], + ): + """Delete all replica disk snapshots. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + :returns: updated volume info, containing snapshot changes. + + Used to delete snapshots that were created while deploying the replica + instance. + """ pass @abc.abstractmethod def restore_replica_disk_snapshots( - self, ctxt, connection_info, target_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + volumes_info: list[dict], + ) -> list[dict]: + """Create new volumes using the snapshots. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param volumes_info: a provider specific list of volumes that must + include the fields defined by + "volumes_info_schema.json". + Populated by "deploy_replica_disks". + :returns: updated volume info, containing the resulting volumes. + + The snapshots created by "create_replica_disk_snapshots" will be used + to create new volumes. The original volumes are expected to be attached + to replica instances at the time of this call. + """ pass class BaseExportInstanceProvider(BaseInstanceProvider): @abc.abstractmethod - def get_source_environment_schema(self): - pass - - -class BaseExportProvider(BaseExportInstanceProvider): + def get_source_environment_schema(self) -> str: + """Retrieve the provider specific source environment schema. - @abc.abstractmethod - def export_instance(self, ctxt, connection_info, source_environment, - instance_name, export_path): - """Exports the given instance. + Users can specify source environment parameters to control the way + in which the instances are exported. - Exports the instance given by its name from the given source cloud - to the provided export directory path using the given connection info. + These settings are provided at run time and usually take precedence + over the coriolis-worker configuration. """ pass @@ -422,42 +826,179 @@ def export_instance(self, ctxt, connection_info, source_environment, class BaseReplicaExportProvider(BaseExportInstanceProvider): @abc.abstractmethod - def get_replica_instance_info(self, ctxt, connection_info, - source_environment, instance_name): + def get_replica_instance_info( + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + instance_name: str, + ) -> dict: + """Get exportable instance info. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param instance_name: the name of the instance to retrieve + :returns: a dict conforming to vm_export_info_schema.json + + TODO: identical to "get_instance", one of them should be deprecated. + """ pass @abc.abstractmethod - def deploy_replica_source_resources(self, ctxt, connection_info, - export_info, source_environment): + def deploy_replica_source_resources( + self, + ctxt: context.RequestContext, + connection_info: dict, + export_info: dict, + source_environment: dict, + ) -> dict: + """Deploys a minion instance to facilitate disk transfers. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param source_environment: provider specific source environment + parameters + :returns: a dict containing the following fields: + * migr_resources: provider specific dict describing the created + resources + * connection_info: minion connection information, conforming to + replication_worker_conn_info_schema.json + + Providers that do not use minion instances to perform disk transfers + can omit this information. + """ pass @abc.abstractmethod - def delete_replica_source_resources(self, ctxt, connection_info, - source_environment, - migr_resources_dict): + def delete_replica_source_resources( + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + migr_resources_dict: dict, + ): + """Cleanup any resources created by "deploy_replica_source_resources". + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param migr_resources_dict: dict returned by + "deploy_replica_source_resources" + """ pass @abc.abstractmethod - def replicate_disks(self, ctxt, connection_info, source_environment, - instance_name, source_resources, source_conn_info, - target_conn_info, volumes_info, incremental): + def replicate_disks( + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + instance_name: str, + source_resources, + source_conn_info: dict, + target_conn_info: dict, + volumes_info: list[bool], + incremental: bool, + ) -> list[dict]: + """Replicate instance disks to the destination platform. + + :param ctxt: Coriolis request context + :param connection_info: source endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param instance_name: the name of the replicated instance + :param source_resources: source minion information returned by + "deploy_replica_source_resources" + :param source_conn_info: source minion connection info, conforming to + replication_worker_conn_info_schema.json + :param target_conn_info: target minion connection info, conforming to + disk_sync_resources_conn_info_schema.json. + Usually returned by the backup writer, + see backup_writers.py. + :param volumes_info: destination volumes conforming to + disk_sync_resources_info_schema.json + :param incremental: perform an incremental transfer, if possible. + Always enabled at the moment. + :returns: the updated destination volume information + + If incremental transfers are supported, consider annotating the + resulting volume info with transfer information. + + WARNING: if minion instances are used on the source side, the provider + is expected to attach the volumes as part of this call, even if it + supports minion pools. "attach_volumes_to_minion" is never called for + source minions. + """ pass @abc.abstractmethod def delete_replica_source_snapshots( - self, ctxt, connection_info, source_environment, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + volumes_info: list[dict], + ): + """Delete any snapshots created during disk replication. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param volumes_info: destination volumes conforming to + disk_sync_resources_info_schema.json + """ pass @abc.abstractmethod - def shutdown_instance(self, ctxt, connection_info, source_environment, - instance_name): + def shutdown_instance( + self, + ctxt: context.RequestContext, + connection_info: dict, + source_environment: dict, + instance_name: str, + ): + """Shutdown the specified instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param source_environment: provider specific source environment + parameters + :param instance_name: the name of the instance to be stopped. + """ pass class BaseInstanceFlavorProvider(BaseProvider): @abc.abstractmethod - def get_optimal_flavor(self, ctxt, connection_info, target_environment, - export_info): + def get_optimal_flavor( + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + export_info: dict, + ) -> str: + """Get the optimal flavor for the exported instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :returns: the destination flavor name + + Some platforms have the concept of flavors, defining the amount of + resources allocated to an instance. + + This method will determine the best suited flavor based on the + specifications of the exported instance. + """ pass @@ -482,9 +1023,24 @@ def get_os_morphing_tools_helper(conn, os_morphing_tools_clss, class BaseEndpointStorageProvider(object, with_metaclass(abc.ABCMeta)): @abc.abstractmethod - def get_storage(self, ctxt, connection_info, target_environment): - """ Returns all the storage options available to the given - credentials within the provided target_environment. + def get_storage( + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + ) -> dict: + """Retrieve information about the available storage backends. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: provider specific target environment + parameters + :returns: a dict conforming to vm_storage_schema.json + + Each entry is expected to contain the name and id of the storage + backend. The user will be allowed to associate source storage backends + with destination storage backends, determining the location and + characteristics of the transferred disks. """ pass @@ -495,8 +1051,14 @@ class BaseUpdateSourceReplicaProvider(object, with_metaclass(abc.ABCMeta)): """ @abc.abstractmethod def check_update_source_environment_params( - self, ctxt, connection_info, instance_name, volumes_info, - old_params, new_params): + self, + ctxt: context.RequestContext, + connection_info: dict, + instance_name: str, + volumes_info: list[dict], + old_params: dict, + new_params: dict, + ) -> list[dict]: """ Checks that any existing replica resources for the VM given by its `export_info` which were replicated using the `old_params` is compatible with the `new_params`. The `old_params` and `new_params` @@ -519,8 +1081,14 @@ class BaseUpdateDestinationReplicaProvider( """ @abc.abstractmethod def check_update_destination_environment_params( - self, ctxt, connection_info, export_info, volumes_info, - old_params, new_params): + self, + ctxt: context.RequestContext, + connection_info: dict, + export_info: dict, + volumes_info: list[dict], + old_params: dict, + new_params: dict, + ) -> list[dict]: """ Checks that any existing replica resources for the VM given by its `export_info` which were replicated using the `old_params` is compatible with the `new_params`. The `old_params` and `new_params` @@ -538,93 +1106,286 @@ def check_update_destination_environment_params( class _BaseMinionPoolProvider( object, with_metaclass(abc.ABCMeta)): - """ Class for providers which offer Minion Pool management functionality. + """Minion Pool management functionality. + + Coriolis users will choose whether minion pools should be used or not. + Even if a provider supports the minion pool functionality, it is still + expected to implement the other methods used for minion management, such + as: + * Import providers (destination side): + * deploy_replica_target_resources + * delete_replica_target_resources + * deploy_os_morphing_resources + * delete_os_morphing_resources + * Export providers (source side): + * deploy_replica_source_resources + * delete_replica_source_resources + + TODO: consider unifying the APIs. """ @abc.abstractmethod - def get_minion_pool_environment_schema(self): - """ Returns the schema for the minion pool options. """ + def get_minion_pool_environment_schema(self) -> str: + """Returns the schema for the minion pool options.""" pass @abc.abstractmethod def get_minion_pool_options( - self, ctxt, connection_info, env=None, option_names=None): - """ Returns possible environment options for minion pools. """ + self, + ctxt: context.RequestContext, + connection_info: dict, + env: dict | None = None, + option_names: list[str] | None = None, + ) -> list[dict]: + """Returns possible environment options for minion pools. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param env: dict: optional target environment options + :param option_names: optional list of parameter names to + show values for + :returns: a list of dicts describing minion pool options. + See "get_target_environment_options" for examples. + """ pass @abc.abstractmethod def validate_minion_compatibility_for_transfer( - self, ctxt, connection_info, export_info, environment_options, - minion_properties): - """ Validates compatibility between the pool's options and the options + self, + ctxt: context.RequestContext, + connection_info: dict, + export_info: dict, + environment_options: list[dict], + minion_properties: dict, + ): + """Validates compatibility between the pool's options and the options selected for a given transfer. Should raise if any options related to the minions in the pool might be deemed incompatible with the desited transfer options. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param environment_options: provider specific pool options + :param minion_properties: minion information + returned by "create_minion" """ pass @abc.abstractmethod def validate_minion_pool_environment_options( - self, ctxt, connection_info, environment_options): - """ Validates the provided pool options. """ + self, + ctxt: context.RequestContext, + connection_info: dict, + environment_options: list[dict], + ): + """Validates the provided pool options. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param environment_options: provider specific pool options + """ pass @abc.abstractmethod def set_up_pool_shared_resources( - self, ctxt, connection_info, environment_options, pool_identifier): - """ Sets up supporting resources which can be re-used amongst the + self, + ctxt: context.RequestContext, + connection_info: dict, + environment_options: list[dict], + pool_identifier: str, + ) -> dict: + """Sets up supporting resources which can be re-used amongst the machines which will be spawned within the pool (e.g. a shared network) + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param environment_options: provider specific pool options + :param pool_identifier: Coriolis allocated pool UUID + :returns: provider specific dict describing the pool shared resources """ pass @abc.abstractmethod def tear_down_pool_shared_resources( - self, ctxt, connection_info, environment_options, - pool_shared_resources): - """ Tears down all pool supporting resources. """ + self, + ctxt: context.RequestContext, + connection_info: dict, + environment_options: list[dict], + pool_shared_resources: dict, + ): + """Tears down all pool supporting resources. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param environment_options: provider specific pool options + :param pool_shared_resources: dict returned by + "set_up_pool_shared_resources" + """ pass @abc.abstractmethod def create_minion( - self, ctxt, connection_info, environment_options, - pool_identifier, pool_os_type, pool_shared_resources, - new_minion_identifier): + self, + ctxt: context.RequestContext, + connection_info: dict, + environment_options: list[dict], + pool_identifier: str, + pool_os_type: str, + pool_shared_resources: dict, + new_minion_identifier: str, + ) -> dict: + """Create a minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param environment_options: provider specific pool options + :param pool_identifier: Coriolis allocated pool UUID + :param pool_os_type: an operating system type such as "linux" or + "windows". See the list of constants for other + OS types. + :param pool_shared_resources: dict returned by + "set_up_pool_shared_resources" + :param new_minion_identifier: Coriolis allocated minion UUID + + :returns: a dict containing the following fields: + * connection_info - minion connection information, conforming to + replication_worker_conn_info_schema.json + * minion_provider_properties - provider specific properties + * backup_writer_connection_info - a dict conforming to + disk_sync_resources_conn_info_schema.json + + The provider is expected to initialize the backup writer. + """ pass @abc.abstractmethod def delete_minion( - self, ctxt, connection_info, minion_properties): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + ): + """Delete the minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + """ pass @abc.abstractmethod def shutdown_minion( - self, ctxt, connection_info, minion_properties): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + ): + """Shutdown the minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + """ pass @abc.abstractmethod def start_minion( - self, ctxt, connection_info, minion_properties): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + ): + """Start the minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + """ pass @abc.abstractmethod def attach_volumes_to_minion( - self, ctxt, connection_info, minion_properties, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + volumes_info: list[dict], + ) -> dict: + """Attach volumes to minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + :param volumes_info: provider specific list of volumes, conforming + to volumes_info_schema.json + :returns: a dict containing the following fields + * minion_properties - updated minion properties + * volumes_info - updated volume info, specifying the attachment + location through "volume_dev" + + See the "deploy_replica_target_resources" description for more details + about the "volume_dev" field. + + WARNING: currently unused on source side, the export providers are + expected to attach the volumes as part of "replicate_disks". + """ pass @abc.abstractmethod def detach_volumes_from_minion( - self, ctxt, connection_info, minion_properties, volumes_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + volumes_info: list[dict], + ): + """Detach volumes from minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + :param volumes_info: provider specific list of volumes, conforming + to volumes_info_schema.json + """ pass @abc.abstractmethod def healthcheck_minion( - self, ctxt, connection_info, - minion_properties, minion_connection_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + minion_properties: dict, + minion_connection_info: dict, + ): + """Verify the state of the minion instance. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". + :param minion_connection_info: minion connection info returned by + "create_minion" + + The provider must raise an exception if the minion instance is not + reachable. + """ pass class BaseSourceMinionPoolProvider(_BaseMinionPoolProvider): - pass @@ -632,20 +1393,64 @@ class BaseDestinationMinionPoolProvider(_BaseMinionPoolProvider): @abc.abstractmethod def validate_osmorphing_minion_compatibility_for_transfer( - self, ctxt, connection_info, export_info, environment_options, - minion_properties): + self, + ctxt: context.RequestContext, + connection_info: dict, + export_info: dict, + environment_options: list[dict], + minion_properties: dict, + ): """ Validates compatibility between the OSMorphing pool's options and the options selected for a given transfer. Should raise if any options of the minions in the pool might be deemed incompatible with the desired transfer options. + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param export_info: a dict describing the exported instance, + conforming to vm_export_info_schema.json + :param environment_options: provider specific pool options + :param minion_properties: provider specific properties returned by + "create_minion" through + "minion_provider_properties". """ pass @abc.abstractmethod def get_additional_os_morphing_info( - self, ctxt, connection_info, target_environment, - instance_deployment_info): + self, + ctxt: context.RequestContext, + connection_info: dict, + target_environment: dict, + instance_deployment_info: dict, + ) -> dict: """ This method should return any additional 'osmorphing_info' as defined in coriolis.schemas.CORIOLIS_OS_MORPHING_RESOURCES_SCHEMA + + :param ctxt: Coriolis request context + :param connection_info: endpoint connection parameters + :param target_environment: user provided target environment parameters + :param instance_deployment_info: target instance information returned + by the provider through the + "deploy_replica_instance" method + """ pass + + +# Unused deprecated classes, kept for backwards compatibility in case there are +# providers that still inherit them. +class BaseMigrationExportValidationProvider: + pass + + +class BaseMigrationImportValidationProvider: + pass + + +class BaseImportProvider(BaseImportInstanceProvider): + pass + + +class BaseExportProvider(BaseExportInstanceProvider): + pass diff --git a/coriolis/providers/factory.py b/coriolis/providers/factory.py index ff3a5cdb..64941f45 100644 --- a/coriolis/providers/factory.py +++ b/coriolis/providers/factory.py @@ -18,10 +18,6 @@ CONF.register_opts(serialization_opts) PROVIDER_TYPE_MAP = { - # NOTE(aznashwan): these have been disabled following the transition from - # classical disk-export-based migrations to Replica-based ones: - # constants.PROVIDER_TYPE_EXPORT: base.BaseExportProvider, - # constants.PROVIDER_TYPE_IMPORT: base.BaseImportProvider, constants.PROVIDER_TYPE_TRANSFER_EXPORT: base.BaseReplicaExportProvider, constants.PROVIDER_TYPE_TRANSFER_IMPORT: base.BaseReplicaImportProvider, constants.PROVIDER_TYPE_ENDPOINT: base.BaseEndpointProvider, @@ -36,12 +32,8 @@ constants.PROVIDER_TYPE_OS_MORPHING: base.BaseImportInstanceProvider, constants.PROVIDER_TYPE_INSTANCE_FLAVOR: base.BaseInstanceFlavorProvider, constants.PROVIDER_TYPE_SETUP_LIBS: base.BaseProviderSetupExtraLibsMixin, - constants.PROVIDER_TYPE_VALIDATE_MIGRATION_EXPORT: ( - base.BaseMigrationExportValidationProvider), constants.PROVIDER_TYPE_VALIDATE_TRANSFER_EXPORT: ( base.BaseReplicaExportValidationProvider), - constants.PROVIDER_TYPE_VALIDATE_MIGRATION_IMPORT: ( - base.BaseMigrationImportValidationProvider), constants.PROVIDER_TYPE_VALIDATE_TRANSFER_IMPORT: ( base.BaseReplicaImportValidationProvider), constants.PROVIDER_TYPE_SOURCE_TRANSFER_UPDATE: ( diff --git a/coriolis/schemas/network_map_schema.json b/coriolis/schemas/network_map_schema.json index ef809bda..1232a7d9 100644 --- a/coriolis/schemas/network_map_schema.json +++ b/coriolis/schemas/network_map_schema.json @@ -1,4 +1,5 @@ { "$schema": "http://cloudbase.it/coriolis/schemas/network_map_schema#", - "type": "object" + "type": "object", + "description": "Mapping between source and destination network identifiers." }