Loading ccserver/jobs.py +78 −66 Original line number Diff line number Diff line Loading @@ -486,10 +486,63 @@ class HotMigrationJob(BaseMigrationJob): if not self._check_status(vm_id, 'running'): raise JobCancelError('vm is not started') to_cleanup = [] # Create storages on destination and start synchronization: disks = vm.get('disk', '').split() for disk in disks: # Getting informations about the disk: to_cleanup += self._sync_disk(vm, disk, source, dest) #... tunres_src = source.proxy.tun_setup() def rb_tun_src(): source.proxy.tun_destroy(tunres_src) self.checkpoint(rb_tun_src) tunres_dst = dest.proxy.tun_setup(local=False) def rb_tun_dst(): dest.proxy.tun_destroy(tunres_dst) self.checkpoint(rb_tun_dst) source.proxy.tun_connect(tunres_src, tunres_dst, dest.get_ip()) dest.proxy.tun_connect_hv(tunres_dst, migration=True) # Initiate the live migration: self.report('migration in progress') source.proxy.vm_migrate_tunneled(self['vm_name'], tunres_src) # At this point, if operation is a success, all we need is just to # cleanup source hypervisor from disk and vm. This operation *CAN'T* # be cancelled or rollbacked if anything fails (unlikely). The # migration must be considered as a success, and the only way to # undo this is to start a new migration in the other way. # Delete the rollback list. # This is mandatory to avoid data loss if the cleanup # code below fail. self.report('cleanup') self._wayback = [] source.proxy.tun_destroy(tunres_src) dest.proxy.tun_destroy(tunres_dst) for cb_cleanup in reversed(to_cleanup): cb_cleanup() # Cleanup the disks: for disk in disks: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) source.proxy.vol_delete(pool, name) logging.info('Job-%s: Migration completed with success', self['id']) def _sync_disk(self, vm, disk, source, dest): to_cleanup = [] # getting informations about the disk: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) size = vm.get('disk%s_size' % disk) Loading @@ -497,22 +550,21 @@ class HotMigrationJob(BaseMigrationJob): assert name is not None, 'name tag doesn\'t exists' assert size is not None, 'size tag doesn\'t exists' self.report('sync volume %s/%s (creation)' % (pool, name)) status_msg = 'sync volume %s/%s (%%s)' % (pool, name) self.report(status_msg % 'creation') # Create the volume on destination: # create the volume on destination: dest.proxy.vol_create(pool, name, int(size)) logging.info('Job-%s: Created volume %s/%s on destination ' logging.info('job-%s: created volume %s/%s on destination ' 'hypervisor', self['id'], pool, name) # Rollback stuff for this action: # rollback stuff for this action: def rb_volcreate(): dest.proxy.vol_delete(pool, name) self.checkpoint(rb_volcreate) # Setup the drbd synchronization with each hypervisors: self.report('sync volume %s/%s (setup)' % (pool, name)) to_cleanup = [] # setup the drbd synchronization with each hypervisors: self.report(status_msg % 'setup') res_src = source.proxy.drbd_setup(pool, name) def rb_setupsrc(): Loading @@ -526,24 +578,27 @@ class HotMigrationJob(BaseMigrationJob): self.checkpoint(rb_setupdst) to_cleanup.append(rb_setupdst) # Start connection of drbd: self.report('sync volume %s/%s (connect)' % (pool, name)) # start connection of drbd: self.report(status_msg % 'connect') source.proxy.drbd_connect(res_src, res_dst, dest.get_ip()) dest.proxy.drbd_connect(res_dst, res_src, source.get_ip()) # Setup topology as Primary/Secondary: # setup topology as primary/secondary: source.proxy.drbd_role(res_src, True) dest.proxy.drbd_role(res_dst, False) # Wait for the end of the disk synchronization: self.report('sync volume %s/%s (sync)' % (pool, name)) cids = set() cids.add(source.connection.async_call('drbd_sync', res_src)) cids.add(dest.connection.async_call('drbd_sync', res_dst)) msgs = self.manager.server.manager.wait(frozenset(cids)) #TODO: check error # Wait the end of the synchronization: sync_running = True while sync_running: status = dest.proxy.drbd_sync_status(res_dst) if status['done']: sync_running = False self.report(status_msg % 'sync %s%%' % status['completion']) time.sleep(2) self.report(status_msg % 'setting both side as primary') source.proxy.drbd_takeover(res_src, True) def rb_takeover_src(): source.proxy.drbd_takeover(res_src, False) Loading @@ -556,50 +611,7 @@ class HotMigrationJob(BaseMigrationJob): self.checkpoint(rb_takeover_dst) to_cleanup.append(rb_takeover_dst) #... tunres_src = source.proxy.tun_setup() def rb_tun_src(): source.proxy.tun_destroy(tunres_src) self.checkpoint(rb_tun_src) tunres_dst = dest.proxy.tun_setup(local=False) def rb_tun_dst(): dest.proxy.tun_destroy(tunres_dst) self.checkpoint(rb_tun_dst) source.proxy.tun_connect(tunres_src, tunres_dst, dest.get_ip()) dest.proxy.tun_connect_hv(tunres_dst, migration=True) # Initiate the live migration: self.report('migration in progress') source.proxy.vm_migrate_tunneled(self['vm_name'], tunres_src) # At this point, if operation is a success, all we need is just to # cleanup source hypervisor from disk and vm. This operation *CAN'T* # be cancelled or rollbacked if anything fails (unlikely). The # migration must be considered as a success, and the only way to # undo this is to start a new migration in the other way. # Delete the rollback list. # This is mandatory to avoid data loss if the cleanup # code below fail. self.report('cleanup') self._wayback = [] source.proxy.tun_destroy(tunres_src) dest.proxy.tun_destroy(tunres_dst) for cb_cleanup in reversed(to_cleanup): cb_cleanup() # Cleanup the disks: for disk in disks: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) source.proxy.vol_delete(pool, name) logging.info('Job-%s: Migration completed with success', self['id']) return to_cleanup class CloneJob(BaseMigrationJob): Loading Loading
ccserver/jobs.py +78 −66 Original line number Diff line number Diff line Loading @@ -486,10 +486,63 @@ class HotMigrationJob(BaseMigrationJob): if not self._check_status(vm_id, 'running'): raise JobCancelError('vm is not started') to_cleanup = [] # Create storages on destination and start synchronization: disks = vm.get('disk', '').split() for disk in disks: # Getting informations about the disk: to_cleanup += self._sync_disk(vm, disk, source, dest) #... tunres_src = source.proxy.tun_setup() def rb_tun_src(): source.proxy.tun_destroy(tunres_src) self.checkpoint(rb_tun_src) tunres_dst = dest.proxy.tun_setup(local=False) def rb_tun_dst(): dest.proxy.tun_destroy(tunres_dst) self.checkpoint(rb_tun_dst) source.proxy.tun_connect(tunres_src, tunres_dst, dest.get_ip()) dest.proxy.tun_connect_hv(tunres_dst, migration=True) # Initiate the live migration: self.report('migration in progress') source.proxy.vm_migrate_tunneled(self['vm_name'], tunres_src) # At this point, if operation is a success, all we need is just to # cleanup source hypervisor from disk and vm. This operation *CAN'T* # be cancelled or rollbacked if anything fails (unlikely). The # migration must be considered as a success, and the only way to # undo this is to start a new migration in the other way. # Delete the rollback list. # This is mandatory to avoid data loss if the cleanup # code below fail. self.report('cleanup') self._wayback = [] source.proxy.tun_destroy(tunres_src) dest.proxy.tun_destroy(tunres_dst) for cb_cleanup in reversed(to_cleanup): cb_cleanup() # Cleanup the disks: for disk in disks: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) source.proxy.vol_delete(pool, name) logging.info('Job-%s: Migration completed with success', self['id']) def _sync_disk(self, vm, disk, source, dest): to_cleanup = [] # getting informations about the disk: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) size = vm.get('disk%s_size' % disk) Loading @@ -497,22 +550,21 @@ class HotMigrationJob(BaseMigrationJob): assert name is not None, 'name tag doesn\'t exists' assert size is not None, 'size tag doesn\'t exists' self.report('sync volume %s/%s (creation)' % (pool, name)) status_msg = 'sync volume %s/%s (%%s)' % (pool, name) self.report(status_msg % 'creation') # Create the volume on destination: # create the volume on destination: dest.proxy.vol_create(pool, name, int(size)) logging.info('Job-%s: Created volume %s/%s on destination ' logging.info('job-%s: created volume %s/%s on destination ' 'hypervisor', self['id'], pool, name) # Rollback stuff for this action: # rollback stuff for this action: def rb_volcreate(): dest.proxy.vol_delete(pool, name) self.checkpoint(rb_volcreate) # Setup the drbd synchronization with each hypervisors: self.report('sync volume %s/%s (setup)' % (pool, name)) to_cleanup = [] # setup the drbd synchronization with each hypervisors: self.report(status_msg % 'setup') res_src = source.proxy.drbd_setup(pool, name) def rb_setupsrc(): Loading @@ -526,24 +578,27 @@ class HotMigrationJob(BaseMigrationJob): self.checkpoint(rb_setupdst) to_cleanup.append(rb_setupdst) # Start connection of drbd: self.report('sync volume %s/%s (connect)' % (pool, name)) # start connection of drbd: self.report(status_msg % 'connect') source.proxy.drbd_connect(res_src, res_dst, dest.get_ip()) dest.proxy.drbd_connect(res_dst, res_src, source.get_ip()) # Setup topology as Primary/Secondary: # setup topology as primary/secondary: source.proxy.drbd_role(res_src, True) dest.proxy.drbd_role(res_dst, False) # Wait for the end of the disk synchronization: self.report('sync volume %s/%s (sync)' % (pool, name)) cids = set() cids.add(source.connection.async_call('drbd_sync', res_src)) cids.add(dest.connection.async_call('drbd_sync', res_dst)) msgs = self.manager.server.manager.wait(frozenset(cids)) #TODO: check error # Wait the end of the synchronization: sync_running = True while sync_running: status = dest.proxy.drbd_sync_status(res_dst) if status['done']: sync_running = False self.report(status_msg % 'sync %s%%' % status['completion']) time.sleep(2) self.report(status_msg % 'setting both side as primary') source.proxy.drbd_takeover(res_src, True) def rb_takeover_src(): source.proxy.drbd_takeover(res_src, False) Loading @@ -556,50 +611,7 @@ class HotMigrationJob(BaseMigrationJob): self.checkpoint(rb_takeover_dst) to_cleanup.append(rb_takeover_dst) #... tunres_src = source.proxy.tun_setup() def rb_tun_src(): source.proxy.tun_destroy(tunres_src) self.checkpoint(rb_tun_src) tunres_dst = dest.proxy.tun_setup(local=False) def rb_tun_dst(): dest.proxy.tun_destroy(tunres_dst) self.checkpoint(rb_tun_dst) source.proxy.tun_connect(tunres_src, tunres_dst, dest.get_ip()) dest.proxy.tun_connect_hv(tunres_dst, migration=True) # Initiate the live migration: self.report('migration in progress') source.proxy.vm_migrate_tunneled(self['vm_name'], tunres_src) # At this point, if operation is a success, all we need is just to # cleanup source hypervisor from disk and vm. This operation *CAN'T* # be cancelled or rollbacked if anything fails (unlikely). The # migration must be considered as a success, and the only way to # undo this is to start a new migration in the other way. # Delete the rollback list. # This is mandatory to avoid data loss if the cleanup # code below fail. self.report('cleanup') self._wayback = [] source.proxy.tun_destroy(tunres_src) dest.proxy.tun_destroy(tunres_dst) for cb_cleanup in reversed(to_cleanup): cb_cleanup() # Cleanup the disks: for disk in disks: pool = vm.get('disk%s_pool' % disk) name = vm.get('disk%s_vol' % disk) source.proxy.vol_delete(pool, name) logging.info('Job-%s: Migration completed with success', self['id']) return to_cleanup class CloneJob(BaseMigrationJob): Loading