diff --git a/ccserver/jobs.py b/ccserver/jobs.py index 390e0fbafa9817eed43a3aacbdfd81fcfad14316..a9b5e26f0f51e8ea97f5f8adcd676156073c3214 100644 --- a/ccserver/jobs.py +++ b/ccserver/jobs.py @@ -486,75 +486,12 @@ class HotMigrationJob(BaseMigrationJob): if not self._check_status(vm_id, 'running'): raise JobCancelError('vm is not started') + to_cleanup = [] + # Create storages on destination and start synchronization: disks = vm.get('disk', '').split() for disk in disks: - # Getting informations about the disk: - pool = vm.get('disk%s_pool' % disk) - name = vm.get('disk%s_vol' % disk) - size = vm.get('disk%s_size' % disk) - assert pool is not None, 'pool tag doesn\'t exists' - assert name is not None, 'name tag doesn\'t exists' - assert size is not None, 'size tag doesn\'t exists' - - self.report('sync volume %s/%s (creation)' % (pool, name)) - - # Create the volume on destination: - dest.proxy.vol_create(pool, name, int(size)) - logging.info('Job-%s: Created volume %s/%s on destination ' - 'hypervisor', self['id'], pool, name) - - # Rollback stuff for this action: - def rb_volcreate(): - dest.proxy.vol_delete(pool, name) - self.checkpoint(rb_volcreate) - - # Setup the drbd synchronization with each hypervisors: - self.report('sync volume %s/%s (setup)' % (pool, name)) - - to_cleanup = [] - - res_src = source.proxy.drbd_setup(pool, name) - def rb_setupsrc(): - source.proxy.drbd_shutdown(res_src) - self.checkpoint(rb_setupsrc) - to_cleanup.append(rb_setupsrc) - - res_dst = dest.proxy.drbd_setup(pool, name) - def rb_setupdst(): - dest.proxy.drbd_shutdown(res_dst) - self.checkpoint(rb_setupdst) - to_cleanup.append(rb_setupdst) - - # Start connection of drbd: - self.report('sync volume %s/%s (connect)' % (pool, name)) - - source.proxy.drbd_connect(res_src, res_dst, dest.get_ip()) - dest.proxy.drbd_connect(res_dst, res_src, source.get_ip()) - - # Setup topology as Primary/Secondary: - source.proxy.drbd_role(res_src, True) - dest.proxy.drbd_role(res_dst, False) - - # Wait for the end of the disk synchronization: - self.report('sync volume %s/%s (sync)' % (pool, name)) - cids = set() - cids.add(source.connection.async_call('drbd_sync', res_src)) - cids.add(dest.connection.async_call('drbd_sync', res_dst)) - msgs = self.manager.server.manager.wait(frozenset(cids)) - #TODO: check error - - source.proxy.drbd_takeover(res_src, True) - def rb_takeover_src(): - source.proxy.drbd_takeover(res_src, False) - self.checkpoint(rb_takeover_src) - to_cleanup.append(rb_takeover_src) - - dest.proxy.drbd_takeover(res_dst, True) - def rb_takeover_dst(): - dest.proxy.drbd_takeover(res_dst, False) - self.checkpoint(rb_takeover_dst) - to_cleanup.append(rb_takeover_dst) + to_cleanup += self._sync_disk(vm, disk, source, dest) #... tunres_src = source.proxy.tun_setup() @@ -602,6 +539,81 @@ class HotMigrationJob(BaseMigrationJob): logging.info('Job-%s: Migration completed with success', self['id']) + def _sync_disk(self, vm, disk, source, dest): + to_cleanup = [] + + # getting informations about the disk: + pool = vm.get('disk%s_pool' % disk) + name = vm.get('disk%s_vol' % disk) + size = vm.get('disk%s_size' % disk) + assert pool is not None, 'pool tag doesn\'t exists' + assert name is not None, 'name tag doesn\'t exists' + assert size is not None, 'size tag doesn\'t exists' + + status_msg = 'sync volume %s/%s (%%s)' % (pool, name) + self.report(status_msg % 'creation') + + # create the volume on destination: + dest.proxy.vol_create(pool, name, int(size)) + logging.info('job-%s: created volume %s/%s on destination ' + 'hypervisor', self['id'], pool, name) + + # rollback stuff for this action: + def rb_volcreate(): + dest.proxy.vol_delete(pool, name) + self.checkpoint(rb_volcreate) + + # setup the drbd synchronization with each hypervisors: + self.report(status_msg % 'setup') + + res_src = source.proxy.drbd_setup(pool, name) + def rb_setupsrc(): + source.proxy.drbd_shutdown(res_src) + self.checkpoint(rb_setupsrc) + to_cleanup.append(rb_setupsrc) + + res_dst = dest.proxy.drbd_setup(pool, name) + def rb_setupdst(): + dest.proxy.drbd_shutdown(res_dst) + self.checkpoint(rb_setupdst) + to_cleanup.append(rb_setupdst) + + # start connection of drbd: + self.report(status_msg % 'connect') + + source.proxy.drbd_connect(res_src, res_dst, dest.get_ip()) + dest.proxy.drbd_connect(res_dst, res_src, source.get_ip()) + + # setup topology as primary/secondary: + source.proxy.drbd_role(res_src, True) + dest.proxy.drbd_role(res_dst, False) + + # Wait the end of the synchronization: + sync_running = True + while sync_running: + status = dest.proxy.drbd_sync_status(res_dst) + if status['done']: + sync_running = False + + self.report(status_msg % 'sync %s%%' % status['completion']) + time.sleep(2) + + self.report(status_msg % 'setting both side as primary') + source.proxy.drbd_takeover(res_src, True) + def rb_takeover_src(): + source.proxy.drbd_takeover(res_src, False) + self.checkpoint(rb_takeover_src) + to_cleanup.append(rb_takeover_src) + + dest.proxy.drbd_takeover(res_dst, True) + def rb_takeover_dst(): + dest.proxy.drbd_takeover(res_dst, False) + self.checkpoint(rb_takeover_dst) + to_cleanup.append(rb_takeover_dst) + + return to_cleanup + + class CloneJob(BaseMigrationJob): '''