From 63cdf9328b3db9c079ab22ebfdd9a2ec3353f8fc Mon Sep 17 00:00:00 2001 From: Chen Zheng <1072122585@qq.com> Date: Mon, 18 Sep 2023 22:19:41 +0800 Subject: [PATCH 1/5] support more image format for image importing: for win, convert imported image to vhdx image format; for mac, convert imported image to qcow2 image format. --- eulerlauncher/backends/mac/image_handler.py | 20 +++++++++++---- eulerlauncher/backends/win/image_handler.py | 28 +++++++++++---------- eulerlauncher/grpcs/client.py | 2 +- eulerlauncher/services/imager_service.py | 6 ++--- eulerlauncher/utils/constants.py | 3 ++- 5 files changed, 36 insertions(+), 23 deletions(-) diff --git a/eulerlauncher/backends/mac/image_handler.py b/eulerlauncher/backends/mac/image_handler.py index f9c8ec6..05e7634 100644 --- a/eulerlauncher/backends/mac/image_handler.py +++ b/eulerlauncher/backends/mac/image_handler.py @@ -98,18 +98,28 @@ class MacImageHandler(object): images['local'][image.name] = image.to_dict() omni_utils.save_json_data(self.image_record_file, images) - if fmt == 'qcow2': - qcow2_name = f'{img_to_load}.qcow2' - shutil.copyfile(path, os.path.join(self.image_dir, qcow2_name)) + if fmt not in constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED: + img_name = f'{img_to_load}.{fmt}' + shutil.copyfile(path, os.path.join(self.image_dir, img_name)) else: # Decompress the image self.LOG.debug(f'Decompressing image file: {path} ...') - qcow2_name = f'{img_to_load}.qcow2' - with open(path, 'rb') as pr, open(os.path.join(self.image_dir, qcow2_name), 'wb') as pw: + fmt = fmt.split('.')[0] + img_name = f'{img_to_load}.{fmt}' + with open(path, 'rb') as pr, open(os.path.join(self.image_dir, img_name), 'wb') as pw: data = pr.read() data_dec = lzma.decompress(data) pw.write(data_dec) + # Convert the img to qcow2 + qcow2_name = img_to_load + '.qcow2' + if fmt != "qcow2": + self.LOG.debug(f'Converting image file: {img_name} to {qcow2_name} ...') + cmd = 'qemu-img convert -O vhdx {0} {1}' + subprocess.call(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name)), shell=True) + self.LOG.debug(f'Cleanup temp files ...') + os.remove(os.path.join(self.image_dir, img_name)) + # Record local image image.path = os.path.join(self.image_dir, qcow2_name) image.status = constants.IMAGE_STATUS_READY diff --git a/eulerlauncher/backends/win/image_handler.py b/eulerlauncher/backends/win/image_handler.py index 9f09f4d..90f1f3a 100644 --- a/eulerlauncher/backends/win/image_handler.py +++ b/eulerlauncher/backends/win/image_handler.py @@ -46,7 +46,7 @@ class WinImageHandler(object): data_dec = lzma.decompress(data) pw.write(data_dec) - # Convert the qcow2 img to vhdx + # Convert the img to vhdx vhdx_name = img_to_download + '.vhdx' self.LOG.debug(f'Converting image file: {img_name} to {vhdx_name} ...') with powershell.PowerShell('GBK') as ps: @@ -94,26 +94,28 @@ class WinImageHandler(object): images['local'][image.name] = image.to_dict() omni_utils.save_json_data(self.image_record_file, images) - if fmt == 'qcow2': - qcow2_name = f'{img_to_load}.qcow2' - shutil.copyfile(path, os.path.join(self.image_dir, qcow2_name)) + if fmt not in constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED: + img_name = f'{img_to_load}.{fmt}' + shutil.copyfile(path, os.path.join(self.image_dir, img_name)) else: # Decompress the image self.LOG.debug(f'Decompressing image file: {path} ...') - qcow2_name = f'{img_to_load}.qcow2' - with open(path, 'rb') as pr, open(os.path.join(self.image_dir, qcow2_name), 'wb') as pw: + fmt = fmt.split('.')[0] + img_name = f'{img_to_load}.{fmt}' + with open(path, 'rb') as pr, open(os.path.join(self.image_dir, img_name), 'wb') as pw: data = pr.read() data_dec = lzma.decompress(data) pw.write(data_dec) - # Convert the qcow2 img to vhdx + # Convert the img to vhdx vhdx_name = img_to_load + '.vhdx' - self.LOG.debug(f'Converting image file: {qcow2_name} to {vhdx_name} ...') - with powershell.PowerShell('GBK') as ps: - cmd = 'qemu-img convert -O vhdx {0} {1}' - outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, qcow2_name), os.path.join(self.image_dir, vhdx_name))) - self.LOG.debug(f'Cleanup temp files ...') - os.remove(os.path.join(self.image_dir, qcow2_name)) + if fmt != "vhdx": + self.LOG.debug(f'Converting image file: {img_name} to {vhdx_name} ...') + with powershell.PowerShell('GBK') as ps: + cmd = 'qemu-img convert -O vhdx {0} {1}' + outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, vhdx_name))) + self.LOG.debug(f'Cleanup temp files ...') + os.remove(os.path.join(self.image_dir, img_name)) # Record local image image.path = os.path.join(self.image_dir, vhdx_name) diff --git a/eulerlauncher/grpcs/client.py b/eulerlauncher/grpcs/client.py index 66adb9a..30a0aa9 100644 --- a/eulerlauncher/grpcs/client.py +++ b/eulerlauncher/grpcs/client.py @@ -50,7 +50,7 @@ class Client(object): return err_msg supported = False - for tp in constants.IMAGE_LOAD_SUPPORTED_TYPES: + for tp in constants.IMAGE_LOAD_SUPPORTED_TYPES + constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED: if path.endswith(tp): supported = True break diff --git a/eulerlauncher/services/imager_service.py b/eulerlauncher/services/imager_service.py index 0d1fb8e..5a53cc3 100644 --- a/eulerlauncher/services/imager_service.py +++ b/eulerlauncher/services/imager_service.py @@ -67,11 +67,11 @@ class ImagerService(images_pb2_grpc.ImageGrpcServiceServicer): LOG.debug(f"Get request to load image: {request.name} from path: {request.path} ...") supported, fmt = omni_utils.check_file_tail( - request.path, omni_constants.IMAGE_LOAD_SUPPORTED_TYPES) + request.path, omni_constants.IMAGE_LOAD_SUPPORTED_TYPES + omni_constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED) if not supported: - supported_fmt = ', '.join(omni_constants.IMAGE_LOAD_SUPPORTED_TYPES) - msg = f'Unsupported image format, the current supported format are: {supported_fmt}.' + supported_fmt = ', '.join(omni_constants.IMAGE_LOAD_SUPPORTED_TYPES + omni_constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED) + msg = f'Unsupported image format, the current supported formats are: {supported_fmt}.' return images_pb2.GeneralImageResponse(ret=1, msg=msg) diff --git a/eulerlauncher/utils/constants.py b/eulerlauncher/utils/constants.py index 9383d2d..3654b79 100644 --- a/eulerlauncher/utils/constants.py +++ b/eulerlauncher/utils/constants.py @@ -14,7 +14,8 @@ IMAGE_STATUS_DOWNLOADING = 'Downloading' IMAGE_STATUS_LOADING = 'Loading' IMAGE_STATUS_READY = 'Ready' -IMAGE_LOAD_SUPPORTED_TYPES = ['qcow2.xz', 'qcow2'] +IMAGE_LOAD_SUPPORTED_TYPES = ['qcow2', 'raw', 'vmdk', 'vhd', 'vhdx', 'qcow', 'vdi'] +IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED = ['qcow2.xz', 'raw.xz', 'vmdk.xz', 'vhd.xz', 'vhdx.xz', 'qcow.xz', 'vdi.xz'] ARCH_MAP = { 'AMD64': 'x86_64', -- Gitee From 154ec7f876f4319e85f5cca7b55d054eaeac19fd Mon Sep 17 00:00:00 2001 From: Chen Zheng <1072122585@qq.com> Date: Tue, 19 Sep 2023 10:07:32 +0800 Subject: [PATCH 2/5] add progress bar for win/mac during image downloading and loading progress --- eulerlauncher/backends/mac/image_handler.py | 33 +++++++++++++++-- eulerlauncher/backends/win/image_handler.py | 41 +++++++++++++++++++-- eulerlauncher/services/imager_service.py | 6 ++- eulerlauncher/utils/utils.py | 7 ++++ requirements-win.txt | 1 + requirements.txt | 1 + 6 files changed, 82 insertions(+), 7 deletions(-) diff --git a/eulerlauncher/backends/mac/image_handler.py b/eulerlauncher/backends/mac/image_handler.py index 05e7634..19820a3 100644 --- a/eulerlauncher/backends/mac/image_handler.py +++ b/eulerlauncher/backends/mac/image_handler.py @@ -40,8 +40,10 @@ class MacImageHandler(object): images['local'][img_to_download] = img_dict omni_utils.save_json_data(self.image_record_file, images) + download_progress_bar_path = os.path.join(self.image_dir, 'download_progress_bar_' + img_to_download) download_cmd = [self.wget_bin, images['remote'][img_to_download]['path'], - '-O', os.path.join(self.image_dir, img_name), '--no-check-certificate'] + '-O', os.path.join(self.image_dir, img_name), '--no-check-certificate', + '--progress=dot:mega', '-q', '--show-progress', f'-o {download_progress_bar_path}'] self.LOG.debug(' '.join(download_cmd)) subprocess.call(' '.join(download_cmd), shell=True) #wget.download(url=images['remote'][img_to_download]['path'], out=os.path.join(self.image_dir, img_name), bar=None) @@ -59,12 +61,24 @@ class MacImageHandler(object): os.remove(os.path.join(self.image_dir, img_name)) # Record local image + if os.path.exists(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)): + os.remove(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)) img_dict['status'] = constants.IMAGE_STATUS_READY img_dict['path'] = os.path.join(self.image_dir, qcow2_name) images['local'][img_to_download] = img_dict omni_utils.save_json_data(self.image_record_file, images) self.LOG.debug(f'Image: {img_to_download} is ready ...') + def download_progress_bar(self, img_name): + progress_bar = [] + progress_bar_path = os.path.join(self.image_dir, 'download_progress_bar_' + img_name) + if os.path.exists(progress_bar_path): + with open(progress_bar_path, "r", encoding=omni_utils.detect_encoding(progress_bar_path), errors='ignore') as progress_bar_file: + progress_bar = progress_bar_file.readlines() + if len(progress_bar) > 1 and progress_bar[-2].strip() != "": + return constants.IMAGE_STATUS_DOWNLOADING + ": " + progress_bar[-2].strip() + else: + return constants.IMAGE_STATUS_DOWNLOADING def delete_image(self, images, img_to_delete): if img_to_delete not in images['local'].keys(): @@ -115,10 +129,12 @@ class MacImageHandler(object): qcow2_name = img_to_load + '.qcow2' if fmt != "qcow2": self.LOG.debug(f'Converting image file: {img_name} to {qcow2_name} ...') - cmd = 'qemu-img convert -O vhdx {0} {1}' - subprocess.call(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name)), shell=True) + load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) + cmd = 'qemu-img convert -p -O qcow2 {0} {1} > {2}' + subprocess.call(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name), load_progress_bar_path), shell=True) self.LOG.debug(f'Cleanup temp files ...') os.remove(os.path.join(self.image_dir, img_name)) + os.remove(load_progress_bar_path) # Record local image image.path = os.path.join(self.image_dir, qcow2_name) @@ -126,3 +142,14 @@ class MacImageHandler(object): images['local'][image.name] = image.to_dict() omni_utils.save_json_data(self.image_record_file, images) self.LOG.debug(f'Image: {qcow2_name} is ready ...') + + def load_progress_bar(self, img_name): + progress_bar_lines = [] + progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_name) + if os.path.exists(progress_bar_path): + with open(progress_bar_path, 'r', encoding=omni_utils.detect_encoding(progress_bar_path), errors='ignore') as progress_bar_file: + progress_bar_lines = progress_bar_file.readlines() + if len(progress_bar_lines) > 1 and progress_bar_lines[-2].strip() != "": + return constants.IMAGE_STATUS_LOADING + ": " + progress_bar_lines[-2].strip() + else: + return constants.IMAGE_STATUS_LOADING \ No newline at end of file diff --git a/eulerlauncher/backends/win/image_handler.py b/eulerlauncher/backends/win/image_handler.py index 90f1f3a..7c6e424 100644 --- a/eulerlauncher/backends/win/image_handler.py +++ b/eulerlauncher/backends/win/image_handler.py @@ -28,6 +28,15 @@ class WinImageHandler(object): # Download the image img_name = wget.filename_from_url(images['remote'][img_to_download]['path']) img_dict = copy.deepcopy(images['remote'][img_to_download]) + downloaded_bytes = 0 + + def progress_bar(current, total, width): + nonlocal downloaded_bytes + if current == 0 or current - downloaded_bytes > 1024*1024 or current == total: + progress_percent = int((current / total) * 100) + downloaded_bytes = current + with open(os.path.join(self.image_dir, 'download_progress_bar_' + img_to_download), 'w') as progress_file: + progress_file.write(f"{current/1024/1024: .2f}/{total/1024/1024: .2f}MB ({progress_percent}%)") if not os.path.exists(os.path.join(self.image_dir, img_name)): self.LOG.debug(f'Downloading image: {img_to_download} from remote repo ...') @@ -35,7 +44,7 @@ class WinImageHandler(object): img_dict['status'] = constants.IMAGE_STATUS_DOWNLOADING images['local'][img_to_download] = img_dict omni_utils.save_json_data(self.image_record_file, images) - wget.download(url=images['remote'][img_to_download]['path'], out=os.path.join(self.image_dir, img_name), bar=None) + wget.download(url=images['remote'][img_to_download]['path'], out=os.path.join(self.image_dir, img_name), bar=progress_bar) self.LOG.debug(f'Image: {img_to_download} succesfully downloaded from remote repo ...') # Decompress the image @@ -55,6 +64,8 @@ class WinImageHandler(object): self.LOG.debug(f'Cleanup temp files ...') os.remove(os.path.join(self.image_dir, qcow2_name)) + if os.path.exists(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)): + os.remove(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)) # Record local image img_dict['status'] = constants.IMAGE_STATUS_READY @@ -63,6 +74,17 @@ class WinImageHandler(object): omni_utils.save_json_data(self.image_record_file, images) self.LOG.debug(f'Image: {img_to_download} is ready ...') + def download_progress_bar(self, img_name): + progress_bar = "" + progress_bar_path = os.path.join(self.image_dir, 'download_progress_bar_' + img_name) + if os.path.exists(progress_bar_path): + with open(progress_bar_path, "r") as progress_bar_file: + progress_bar = progress_bar_file.read() + if progress_bar != "": + return constants.IMAGE_STATUS_DOWNLOADING + ": " + progress_bar + else: + return constants.IMAGE_STATUS_DOWNLOADING + def delete_image(self, images, img_to_delete): if img_to_delete not in images['local'].keys(): return 1 @@ -111,11 +133,13 @@ class WinImageHandler(object): vhdx_name = img_to_load + '.vhdx' if fmt != "vhdx": self.LOG.debug(f'Converting image file: {img_name} to {vhdx_name} ...') + load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) with powershell.PowerShell('GBK') as ps: - cmd = 'qemu-img convert -O vhdx {0} {1}' - outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, vhdx_name))) + cmd = 'qemu-img convert -p -O vhdx {0} {1} | Out-File -FilePath {2}' + outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, vhdx_name), load_progress_bar_path)) self.LOG.debug(f'Cleanup temp files ...') os.remove(os.path.join(self.image_dir, img_name)) + os.remove(load_progress_bar_path) # Record local image image.path = os.path.join(self.image_dir, vhdx_name) @@ -123,3 +147,14 @@ class WinImageHandler(object): images['local'][image.name] = image.to_dict() omni_utils.save_json_data(self.image_record_file, images) self.LOG.debug(f'Image: {vhdx_name} is ready ...') + + def load_progress_bar(self, img_name): + progress_bar_lines = [] + progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_name) + if os.path.exists(progress_bar_path): + with open(progress_bar_path, 'r', encoding=omni_utils.detect_encoding(progress_bar_path), errors='ignore') as progress_bar_file: + progress_bar_lines = progress_bar_file.readlines() + if len(progress_bar_lines) > 1 and progress_bar_lines[-2].strip() != "": + return constants.IMAGE_STATUS_LOADING + ": " + progress_bar_lines[-2].strip() + else: + return constants.IMAGE_STATUS_LOADING \ No newline at end of file diff --git a/eulerlauncher/services/imager_service.py b/eulerlauncher/services/imager_service.py index 5a53cc3..6f8872a 100644 --- a/eulerlauncher/services/imager_service.py +++ b/eulerlauncher/services/imager_service.py @@ -41,6 +41,10 @@ class ImagerService(images_pb2_grpc.ImageGrpcServiceServicer): image.name = img['name'] image.location = img['location'] image.status = img['status'] + if image.status == omni_constants.IMAGE_STATUS_DOWNLOADING: + image.status = self.backend.download_progress_bar(image.name) + elif image.status == omni_constants.IMAGE_STATUS_LOADING: + image.status = self.backend.load_progress_bar(image.name) ret.append(image) LOG.debug(f"Responded: {ret}") return images_pb2.ListImageResponse(images=ret) @@ -51,7 +55,7 @@ class ImagerService(images_pb2_grpc.ImageGrpcServiceServicer): if request.name not in all_images['remote'].keys(): LOG.debug(f'Image: {request.name} not valid for download') - msg = f'Error: Image {request.name} is valid for download, please check image name from REMOTE IMAGE LIST using "images" command ...' + msg = f'Error: Image {request.name} is not valid for download, please check image name from REMOTE IMAGE LIST using "images" command ...' return images_pb2.GeneralImageResponse(ret=1, msg=msg) @omni_utils.asyncwrapper diff --git a/eulerlauncher/utils/utils.py b/eulerlauncher/utils/utils.py index 8381c74..3503fd8 100644 --- a/eulerlauncher/utils/utils.py +++ b/eulerlauncher/utils/utils.py @@ -4,6 +4,7 @@ import os import random from threading import Thread import uuid +import chardet from google.protobuf.json_format import MessageToDict @@ -92,3 +93,9 @@ def check_file_tail(file_name, to_check): break return ret, ret_fmt + +def detect_encoding(file_path): + with open(file_path, 'rb') as file: + raw_data = file.read() + result = chardet.detect(raw_data) + return result['encoding'] \ No newline at end of file diff --git a/requirements-win.txt b/requirements-win.txt index b67b36d..f530041 100644 --- a/requirements-win.txt +++ b/requirements-win.txt @@ -1,6 +1,7 @@ argparse click configparser +chardet dnspython grpcio grpcio-tools diff --git a/requirements.txt b/requirements.txt index db5e0f6..297a362 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ argparse click configparser +chardet dnspython grpcio grpcio-tools -- Gitee From d6c9db4982b365ed136b6eeacbdbe89040dd699a Mon Sep 17 00:00:00 2001 From: Chen Zheng <1072122585@qq.com> Date: Thu, 28 Sep 2023 21:32:18 +0800 Subject: [PATCH 3/5] add take/export snapshot function and export python/go/java development image function --- docs/mac-user-manual.md | 5 +- etc/eulerlauncher.conf | 1 + eulerlauncher/backends/mac/image_handler.py | 5 +- .../backends/mac/instance_handler.py | 62 ++++++++++++++++- eulerlauncher/backends/mac/qemu.py | 14 ++++ .../backends/win/instance_handler.py | 34 ++++++++++ eulerlauncher/backends/win/vmops.py | 7 ++ eulerlauncher/cli.py | 29 ++++++++ eulerlauncher/grpcs/client.py | 14 ++++ .../grpcs/eulerlauncher_grpc/instances.proto | 23 +++++++ .../grpcs/eulerlauncher_grpc/instances_pb2.py | 49 ++++++++------ .../eulerlauncher_grpc/instances_pb2_grpc.py | 66 +++++++++++++++++++ eulerlauncher/grpcs/instances.py | 12 ++++ eulerlauncher/services/instance_service.py | 30 ++++++++- requirements-win.txt | 3 +- requirements.txt | 3 +- 16 files changed, 328 insertions(+), 29 deletions(-) diff --git a/docs/mac-user-manual.md b/docs/mac-user-manual.md index f7d5771..a2c0698 100644 --- a/docs/mac-user-manual.md +++ b/docs/mac-user-manual.md @@ -58,15 +58,17 @@ brew install wget 2. 配置**EulerLauncher**: - - 查看`qemu`及`wget`所处位置,`qemu`二进制文件在不同架构下名称不同,请根据自身情况选择正确的名称(Apple Silicon: qemu-system-aarch64; Intel: qemu-system-x86_64): + - 查看`qemu`,`qemu-img`及`wget`所处位置,`qemu`二进制文件在不同架构下名称不同,请根据自身情况选择正确的名称(Apple Silicon: qemu-system-aarch64; Intel: qemu-system-x86_64): ``` Shell which wget which qemu-system-{host_arch} + which qemu-img ``` 参考输出: ``` /opt/homebrew/bin/wget /opt/homebrew/bin/qemu-system-aarch64 + /opt/homebrew/bin/qemu-img ``` 查看完成后,记录路径结果,在接下来的步骤中将会使用到。 @@ -82,6 +84,7 @@ brew install wget work_dir = # eulerlauncher工作目录,用于存储虚拟机镜像、虚拟机文件等 wget_dir = # wget的可执行文件路径,请参考上一步的内容进行配置 qemu_dir = # qemu的可执行文件路径,请参考上一步的内容进行配置 + qemu_img_dir = # qemu-img的可执行文件路径,请参考上一步的内容进行配置 debug = True [vm] diff --git a/etc/eulerlauncher.conf b/etc/eulerlauncher.conf index a101a51..dcac7da 100644 --- a/etc/eulerlauncher.conf +++ b/etc/eulerlauncher.conf @@ -3,6 +3,7 @@ log_dir = work_dir = wget_dir = qemu_dir = +qemu_img_dir = debug = True [vm] diff --git a/eulerlauncher/backends/mac/image_handler.py b/eulerlauncher/backends/mac/image_handler.py index 19820a3..122ac76 100644 --- a/eulerlauncher/backends/mac/image_handler.py +++ b/eulerlauncher/backends/mac/image_handler.py @@ -24,6 +24,7 @@ class MacImageHandler(object): self.image_record_file = image_record_file self.base_dir = base_dir self.wget_bin = conf.conf.get('default', 'wget_dir') + self.qemu_img_bin = conf.conf.get('default', 'qemu_img_dir') self.LOG = logger @@ -130,8 +131,8 @@ class MacImageHandler(object): if fmt != "qcow2": self.LOG.debug(f'Converting image file: {img_name} to {qcow2_name} ...') load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) - cmd = 'qemu-img convert -p -O qcow2 {0} {1} > {2}' - subprocess.call(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name), load_progress_bar_path), shell=True) + cmd = 'sudo {0} convert -p -O qcow2 {1} {2} > {3}' + subprocess.call(cmd.format(self.qemu_img_bin, os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name), load_progress_bar_path), shell=True) self.LOG.debug(f'Cleanup temp files ...') os.remove(os.path.join(self.image_dir, img_name)) os.remove(load_progress_bar_path) diff --git a/eulerlauncher/backends/mac/instance_handler.py b/eulerlauncher/backends/mac/instance_handler.py index a53e78d..8225e4b 100644 --- a/eulerlauncher/backends/mac/instance_handler.py +++ b/eulerlauncher/backends/mac/instance_handler.py @@ -5,6 +5,7 @@ import signal import subprocess import sys import time +import paramiko from oslo_utils import uuidutils @@ -100,16 +101,21 @@ class MacInstanceHandler(object): except KeyError: return 0 - def create_instance(self, name, image_id, instance_record, all_instances, all_images): + def create_instance(self, name, image_id, instance_record, all_instances, all_images, is_same, mac_address, uuid): # Create dir for the instance - vm_uuid = uuidutils.generate_uuid() + if not is_same: + vm_uuid = uuidutils.generate_uuid() + vm_mac_address = omni_utils.generate_mac() + else: + vm_uuid = uuid + vm_mac_address = mac_address vm_dict = { 'name': name, 'uuid': vm_uuid, 'image': image_id, 'vm_state': constants.VM_STATE_MAP[99], 'ip_address': 'N/A', - 'mac_address': omni_utils.generate_mac(), + 'mac_address': vm_mac_address, 'identification': { 'type': 'pid', 'id': None @@ -175,3 +181,53 @@ class MacInstanceHandler(object): return 0 + def _get_vm_img_id_by_name(self, name): + instance = omni_utils.load_json_data(self.instance_record_file)['instances'][name] + return instance['image'] + + def take_snapshot(self, name, snapshot_name, dest_path, all_instances, all_images, instance_record): + vm_img_id = self._get_vm_img_id_by_name(name) + vm_root_disk_src_path = os.path.join(self.instance_dir, name, vm_img_id + '.qcow2') + vm_root_disk_dst_path = os.path.join(self.instance_dir, vm_img_id + '.qcow2') + mac_address = all_instances['instances'][name]['mac_address'] + vm_uuid = all_instances['instances'][name]['uuid'] + # shutdown the vm first for taking snapshot + shutil.copyfile(vm_root_disk_src_path, vm_root_disk_dst_path) + self.delete_instance(name, instance_record, all_instances) + self.driver.take_and_export_snapshot(snapshot_name, vm_root_disk_dst_path, snapshot_name, dest_path) + # a little hack here, since the running vm's image is already deleted, change the local image path to the copyed image path + all_images['local'][vm_img_id]['path'] = vm_root_disk_dst_path + self.create_instance(name, vm_img_id, instance_record, all_instances, all_images, True, mac_address, vm_uuid) + os.remove(vm_root_disk_dst_path) + return os.path.join(dest_path, f'{snapshot_name}.qcow2') + + def _get_vm_ip_by_name(self, name): + instance = omni_utils.load_json_data(self.instance_record_file)['instances'][name] + if not instance['ip_address']: + ip_address = self._parse_ip_addr(instance['mac_address']) + else: + ip_address = instance['ip_address'] + return ip_address + + def make_development_image(self, name, pwd): + ssh_client = paramiko.SSHClient() + try: + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(self._get_vm_ip_by_name(name), 22, "root", pwd) + bash_command = """ + if which apt >/dev/null 2>&1; + then apt install python3-dev golang openjdk-11-jdk + elif which yum >/dev/null 2>&1; + then yum install -y python3-devel golang java-11-openjdk-devel + elif which dnf >/dev/null 2>&1; + then dnf install -y python3-devel golang java-11-openjdk-devel + fi + """ + stdin, stdout, stderr = ssh_client.exec_command(bash_command) + + self.LOG.debug(stdout.read().decode()) + ssh_client.close() + return 0 + except Exception as e: + self.LOG.debug(f"install development environment failed: {str(e)}") + return 1 \ No newline at end of file diff --git a/eulerlauncher/backends/mac/qemu.py b/eulerlauncher/backends/mac/qemu.py index 2bdd900..1824187 100644 --- a/eulerlauncher/backends/mac/qemu.py +++ b/eulerlauncher/backends/mac/qemu.py @@ -11,6 +11,7 @@ class QemuDriver(object): host_arch_raw = platform.uname().machine host_arch = constants.ARCH_MAP[host_arch_raw] self.qemu_bin = conf.conf.get('default', 'qemu_dir') + self.qemu_img_bin = conf.conf.get('default', 'qemu_img_dir') self.uefi_file = os.path.join('/Library/Application\ Support/org.openeuler.eulerlauncher/','edk2-' + host_arch + '-code.fd') self.uefi_params = ',if=pflash,format=raw,readonly=on' self.vm_cpu = conf.conf.get('vm', 'cpu_num') @@ -28,3 +29,16 @@ class QemuDriver(object): self.LOG.debug(' '.join(qemu_cmd)) instance_process = subprocess.Popen(' '.join(qemu_cmd), shell=True) return instance_process + + def take_and_export_snapshot(self, snapshot_name, vm_root_disk, export_image_name, dest_path): + take_snapshot_cmd = [ + 'sudo', self.qemu_img_bin, 'snapshot', '-c', f'{snapshot_name}', f'{vm_root_disk}' + ] + self.LOG.debug(' '.join(take_snapshot_cmd)) + subprocess.call(' '.join(take_snapshot_cmd), shell=True) + export_image_cmd = [ + 'sudo', self.qemu_img_bin, 'convert', '-l', f'snapshot.name={snapshot_name}', '-O', + 'qcow2', f'{vm_root_disk}', os.path.join(dest_path, f'{export_image_name}.qcow2') + ] + self.LOG.debug(' '.join(export_image_cmd)) + subprocess.call(' '.join(export_image_cmd), shell=True) \ No newline at end of file diff --git a/eulerlauncher/backends/win/instance_handler.py b/eulerlauncher/backends/win/instance_handler.py index 7b27055..70c49e1 100644 --- a/eulerlauncher/backends/win/instance_handler.py +++ b/eulerlauncher/backends/win/instance_handler.py @@ -1,6 +1,8 @@ import os import shutil import time +import paramiko +from glob import glob from oslo_utils import uuidutils from os_win import constants as os_win_const @@ -198,3 +200,35 @@ class WinInstanceHandler(object): """Power on the specified instance.""" self.LOG.debug("Power on instance", instance=instance) self._set_vm_state(instance, os_win_const.HYPERV_VM_STATE_ENABLED) + + def take_snapshot(self, name, snapshot_name, dest_path, all_instances, all_images, instance_record): + _vmops.take_snapshot(name, snapshot_name) + _vmops.export_vm(name, os.path.join(dest_path)) + os.rename(glob(os.path.join(dest_path, name, 'Virtual Hard Disks', '*.vhdx'))[0], os.path.join(dest_path, name, 'Virtual Hard Disks', f'{snapshot_name}.vhdx')) + shutil.move(os.path.join(dest_path, name, 'Virtual Hard Disks', f'{snapshot_name}.vhdx'), os.path.join(dest_path)) + # remove the exported file dir + shutil.rmtree(os.path.join(dest_path, name)) + return os.path.join(dest_path, f'{snapshot_name}.vhdx') + + def make_development_image(self, name, pwd): + ssh_client = paramiko.SSHClient() + try: + ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + ssh_client.connect(_vmops.get_instance_ip_addr(name), 22, "root", pwd) + bash_command = """ + if which apt >/dev/null 2>&1; + then apt install python3-dev golang openjdk-11-jdk + elif which yum >/dev/null 2>&1; + then yum install -y python3-devel golang java-11-openjdk-devel + elif which dnf >/dev/null 2>&1; + then dnf install -y python3-devel golang java-11-openjdk-devel + fi + """ + stdin, stdout, stderr = ssh_client.exec_command(bash_command) + + self.LOG.debug(stdout.read().decode()) + ssh_client.close() + return 0 + except Exception as e: + self.LOG.debug(f"install development environment failed: {str(e)}") + return 1 \ No newline at end of file diff --git a/eulerlauncher/backends/win/vmops.py b/eulerlauncher/backends/win/vmops.py index b8e4c75..dbe8672 100644 --- a/eulerlauncher/backends/win/vmops.py +++ b/eulerlauncher/backends/win/vmops.py @@ -33,6 +33,7 @@ class VMOps(object): def __init__(self, virtapi=None): self._virtapi = virtapi self._vmutils = VMUtils_omni() + self._migrationutils = utilsfactory.get_migrationutils() self._netutils = utilsfactory.get_networkutils() self._hostutils = utilsfactory.get_hostutils() @@ -181,3 +182,9 @@ class VMOps(object): def get_host_ips(self): return self._hostutils.get_local_ips() + + def take_snapshot(self, vm_name, snapshot_name): + self._vmutils.take_vm_snapshot(vm_name, snapshot_name) + + def export_vm(self, vm_name, dest_path): + self._migrationutils.export_vm(vm_name, dest_path, copy_snapshots_config=os_win_const.EXPORT_CONFIG_NO_SNAPSHOTS, copy_vm_storage=True, create_export_subdir=True) \ No newline at end of file diff --git a/eulerlauncher/cli.py b/eulerlauncher/cli.py index 56972a1..dd10af5 100644 --- a/eulerlauncher/cli.py +++ b/eulerlauncher/cli.py @@ -1,5 +1,6 @@ import click import prettytable as pt +import getpass from eulerlauncher.grpcs import client @@ -125,6 +126,32 @@ def launch(vm_name, image): else: print(ret['msg']) +@click.command() +@click.argument('vm_name') +@click.option('--snapshot_name', help='name for the snapshot image') +@click.option('--export_path', help='path for the exported snapshot image') +def take_snapshot(vm_name, snapshot_name, export_path): + + try: + ret = launcher_client.take_snapshot(vm_name, snapshot_name, export_path) + except Exception: + print('Calling to EulerLauncherd daemon failed, please check EulerLauncherd daemon status ...') + else: + print(ret['msg']) + +@click.command() +@click.argument('vm_name') +@click.option('--image_name', help='name for the Python/Go/Java development image') +@click.option('--export_path', help='path for the exported Python/Go/Java development image') +def export_development_image(vm_name, image_name, export_path): + + try: + pwd = getpass.getpass(f"Password for vm[{vm_name}] as root: ") + ret = launcher_client.export_development_image(vm_name, image_name, export_path, pwd) + except Exception: + print('Calling to EulerLauncherd daemon failed, please check EulerLauncherd daemon status ...') + else: + print(ret['msg']) @click.group() def cli(): @@ -139,4 +166,6 @@ if __name__ == '__main__': cli.add_command(launch) cli.add_command(delete_image) cli.add_command(delete_instance) + cli.add_command(take_snapshot) + cli.add_command(export_development_image) cli() \ No newline at end of file diff --git a/eulerlauncher/grpcs/client.py b/eulerlauncher/grpcs/client.py index 30a0aa9..4fee29c 100644 --- a/eulerlauncher/grpcs/client.py +++ b/eulerlauncher/grpcs/client.py @@ -93,3 +93,17 @@ class Client(object): """ return self._instances.delete(name) + + @omnivirt_utils.response2dict + def take_snapshot(self, vm_name, snapshot_name, export_path): + """ Take snapshot + """ + + return self._instances.take_snapshot(vm_name, snapshot_name, export_path) + + @omnivirt_utils.response2dict + def export_development_image(self, vm_name, image_name, export_path, pwd): + """ Export Python/Go/Java development image + """ + + return self._instances.export_development_image(vm_name, image_name, export_path, pwd) \ No newline at end of file diff --git a/eulerlauncher/grpcs/eulerlauncher_grpc/instances.proto b/eulerlauncher/grpcs/eulerlauncher_grpc/instances.proto index c2e751c..b00d88f 100644 --- a/eulerlauncher/grpcs/eulerlauncher_grpc/instances.proto +++ b/eulerlauncher/grpcs/eulerlauncher_grpc/instances.proto @@ -8,6 +8,8 @@ service InstanceGrpcService { rpc list_instances (ListInstancesRequest) returns (ListInstancesResponse) {} rpc create_instance (CreateInstanceRequest) returns (CreateInstanceResponse) {} rpc delete_instance (DeleteInstanceRequest) returns (DeleteInstanceResponse) {} + rpc take_snapshot (TakeSnapshotRequest) returns (TakeSnapshotResponse) {} + rpc export_development_image (ExportDevelopmentImageRequest) returns (ExportDevelopmentImageResponse) {} } @@ -46,3 +48,24 @@ message DeleteInstanceResponse { uint32 ret = 1; string msg = 2; } + +message TakeSnapshotRequest { + string name = 1; + string snapshot = 2; + string dest_path = 3; +} + +message TakeSnapshotResponse { + string msg = 1; +} + +message ExportDevelopmentImageRequest { + string name = 1; + string image = 2; + string dest_path = 3; + string pwd = 4; +} + +message ExportDevelopmentImageResponse { + string msg = 1; +} \ No newline at end of file diff --git a/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2.py b/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2.py index 9877833..e3b60c1 100644 --- a/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2.py +++ b/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2.py @@ -2,10 +2,10 @@ # Generated by the protocol buffer compiler. DO NOT EDIT! # source: instances.proto """Generated protocol buffer code.""" -from google.protobuf.internal import builder as _builder from google.protobuf import descriptor as _descriptor from google.protobuf import descriptor_pool as _descriptor_pool from google.protobuf import symbol_database as _symbol_database +from google.protobuf.internal import builder as _builder # @@protoc_insertion_point(imports) _sym_db = _symbol_database.Default() @@ -13,28 +13,37 @@ _sym_db = _symbol_database.Default() -DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0finstances.proto\x12\x08omnivirt\"M\n\x08Instance\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x10\n\x08vm_state\x18\x03 \x01(\t\x12\x12\n\nip_address\x18\x04 \x01(\t\"\x16\n\x14ListInstancesRequest\">\n\x15ListInstancesResponse\x12%\n\tinstances\x18\x01 \x03(\x0b\x32\x12.omnivirt.Instance\"4\n\x15\x43reateInstanceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\"j\n\x16\x43reateInstanceResponse\x12\x0b\n\x03ret\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12)\n\x08instance\x18\x03 \x01(\x0b\x32\x12.omnivirt.InstanceH\x00\x88\x01\x01\x42\x0b\n\t_instance\"%\n\x15\x44\x65leteInstanceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x16\x44\x65leteInstanceResponse\x12\x0b\n\x03ret\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t2\x9a\x02\n\x13InstanceGrpcService\x12S\n\x0elist_instances\x12\x1e.omnivirt.ListInstancesRequest\x1a\x1f.omnivirt.ListInstancesResponse\"\x00\x12V\n\x0f\x63reate_instance\x12\x1f.omnivirt.CreateInstanceRequest\x1a .omnivirt.CreateInstanceResponse\"\x00\x12V\n\x0f\x64\x65lete_instance\x12\x1f.omnivirt.DeleteInstanceRequest\x1a .omnivirt.DeleteInstanceResponse\"\x00\x42\x03\x80\x01\x01\x62\x06proto3') +DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(b'\n\x0finstances.proto\x12\x08omnivirt\"M\n\x08Instance\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x10\n\x08vm_state\x18\x03 \x01(\t\x12\x12\n\nip_address\x18\x04 \x01(\t\"\x16\n\x14ListInstancesRequest\">\n\x15ListInstancesResponse\x12%\n\tinstances\x18\x01 \x03(\x0b\x32\x12.omnivirt.Instance\"4\n\x15\x43reateInstanceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\"j\n\x16\x43reateInstanceResponse\x12\x0b\n\x03ret\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\x12)\n\x08instance\x18\x03 \x01(\x0b\x32\x12.omnivirt.InstanceH\x00\x88\x01\x01\x42\x0b\n\t_instance\"%\n\x15\x44\x65leteInstanceRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\"2\n\x16\x44\x65leteInstanceResponse\x12\x0b\n\x03ret\x18\x01 \x01(\r\x12\x0b\n\x03msg\x18\x02 \x01(\t\"H\n\x13TakeSnapshotRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\x10\n\x08snapshot\x18\x02 \x01(\t\x12\x11\n\tdest_path\x18\x03 \x01(\t\"#\n\x14TakeSnapshotResponse\x12\x0b\n\x03msg\x18\x01 \x01(\t\"\\\n\x1d\x45xportDevelopmentImageRequest\x12\x0c\n\x04name\x18\x01 \x01(\t\x12\r\n\x05image\x18\x02 \x01(\t\x12\x11\n\tdest_path\x18\x03 \x01(\t\x12\x0b\n\x03pwd\x18\x04 \x01(\t\"-\n\x1e\x45xportDevelopmentImageResponse\x12\x0b\n\x03msg\x18\x01 \x01(\t2\xdd\x03\n\x13InstanceGrpcService\x12S\n\x0elist_instances\x12\x1e.omnivirt.ListInstancesRequest\x1a\x1f.omnivirt.ListInstancesResponse\"\x00\x12V\n\x0f\x63reate_instance\x12\x1f.omnivirt.CreateInstanceRequest\x1a .omnivirt.CreateInstanceResponse\"\x00\x12V\n\x0f\x64\x65lete_instance\x12\x1f.omnivirt.DeleteInstanceRequest\x1a .omnivirt.DeleteInstanceResponse\"\x00\x12P\n\rtake_snapshot\x12\x1d.omnivirt.TakeSnapshotRequest\x1a\x1e.omnivirt.TakeSnapshotResponse\"\x00\x12o\n\x18\x65xport_development_image\x12\'.omnivirt.ExportDevelopmentImageRequest\x1a(.omnivirt.ExportDevelopmentImageResponse\"\x00\x42\x03\x80\x01\x01\x62\x06proto3') -_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals()) -_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'instances_pb2', globals()) +_globals = globals() +_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, _globals) +_builder.BuildTopDescriptorsAndMessages(DESCRIPTOR, 'instances_pb2', _globals) if _descriptor._USE_C_DESCRIPTORS == False: DESCRIPTOR._options = None DESCRIPTOR._serialized_options = b'\200\001\001' - _INSTANCE._serialized_start=29 - _INSTANCE._serialized_end=106 - _LISTINSTANCESREQUEST._serialized_start=108 - _LISTINSTANCESREQUEST._serialized_end=130 - _LISTINSTANCESRESPONSE._serialized_start=132 - _LISTINSTANCESRESPONSE._serialized_end=194 - _CREATEINSTANCEREQUEST._serialized_start=196 - _CREATEINSTANCEREQUEST._serialized_end=248 - _CREATEINSTANCERESPONSE._serialized_start=250 - _CREATEINSTANCERESPONSE._serialized_end=356 - _DELETEINSTANCEREQUEST._serialized_start=358 - _DELETEINSTANCEREQUEST._serialized_end=395 - _DELETEINSTANCERESPONSE._serialized_start=397 - _DELETEINSTANCERESPONSE._serialized_end=447 - _INSTANCEGRPCSERVICE._serialized_start=450 - _INSTANCEGRPCSERVICE._serialized_end=732 + _globals['_INSTANCE']._serialized_start=29 + _globals['_INSTANCE']._serialized_end=106 + _globals['_LISTINSTANCESREQUEST']._serialized_start=108 + _globals['_LISTINSTANCESREQUEST']._serialized_end=130 + _globals['_LISTINSTANCESRESPONSE']._serialized_start=132 + _globals['_LISTINSTANCESRESPONSE']._serialized_end=194 + _globals['_CREATEINSTANCEREQUEST']._serialized_start=196 + _globals['_CREATEINSTANCEREQUEST']._serialized_end=248 + _globals['_CREATEINSTANCERESPONSE']._serialized_start=250 + _globals['_CREATEINSTANCERESPONSE']._serialized_end=356 + _globals['_DELETEINSTANCEREQUEST']._serialized_start=358 + _globals['_DELETEINSTANCEREQUEST']._serialized_end=395 + _globals['_DELETEINSTANCERESPONSE']._serialized_start=397 + _globals['_DELETEINSTANCERESPONSE']._serialized_end=447 + _globals['_TAKESNAPSHOTREQUEST']._serialized_start=449 + _globals['_TAKESNAPSHOTREQUEST']._serialized_end=521 + _globals['_TAKESNAPSHOTRESPONSE']._serialized_start=523 + _globals['_TAKESNAPSHOTRESPONSE']._serialized_end=558 + _globals['_EXPORTDEVELOPMENTIMAGEREQUEST']._serialized_start=560 + _globals['_EXPORTDEVELOPMENTIMAGEREQUEST']._serialized_end=652 + _globals['_EXPORTDEVELOPMENTIMAGERESPONSE']._serialized_start=654 + _globals['_EXPORTDEVELOPMENTIMAGERESPONSE']._serialized_end=699 + _globals['_INSTANCEGRPCSERVICE']._serialized_start=702 + _globals['_INSTANCEGRPCSERVICE']._serialized_end=1179 # @@protoc_insertion_point(module_scope) diff --git a/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2_grpc.py b/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2_grpc.py index 970781c..f895a15 100644 --- a/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2_grpc.py +++ b/eulerlauncher/grpcs/eulerlauncher_grpc/instances_pb2_grpc.py @@ -29,6 +29,16 @@ class InstanceGrpcServiceStub(object): request_serializer=instances__pb2.DeleteInstanceRequest.SerializeToString, response_deserializer=instances__pb2.DeleteInstanceResponse.FromString, ) + self.take_snapshot = channel.unary_unary( + '/omnivirt.InstanceGrpcService/take_snapshot', + request_serializer=instances__pb2.TakeSnapshotRequest.SerializeToString, + response_deserializer=instances__pb2.TakeSnapshotResponse.FromString, + ) + self.export_development_image = channel.unary_unary( + '/omnivirt.InstanceGrpcService/export_development_image', + request_serializer=instances__pb2.ExportDevelopmentImageRequest.SerializeToString, + response_deserializer=instances__pb2.ExportDevelopmentImageResponse.FromString, + ) class InstanceGrpcServiceServicer(object): @@ -52,6 +62,18 @@ class InstanceGrpcServiceServicer(object): context.set_details('Method not implemented!') raise NotImplementedError('Method not implemented!') + def take_snapshot(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + + def export_development_image(self, request, context): + """Missing associated documentation comment in .proto file.""" + context.set_code(grpc.StatusCode.UNIMPLEMENTED) + context.set_details('Method not implemented!') + raise NotImplementedError('Method not implemented!') + def add_InstanceGrpcServiceServicer_to_server(servicer, server): rpc_method_handlers = { @@ -70,6 +92,16 @@ def add_InstanceGrpcServiceServicer_to_server(servicer, server): request_deserializer=instances__pb2.DeleteInstanceRequest.FromString, response_serializer=instances__pb2.DeleteInstanceResponse.SerializeToString, ), + 'take_snapshot': grpc.unary_unary_rpc_method_handler( + servicer.take_snapshot, + request_deserializer=instances__pb2.TakeSnapshotRequest.FromString, + response_serializer=instances__pb2.TakeSnapshotResponse.SerializeToString, + ), + 'export_development_image': grpc.unary_unary_rpc_method_handler( + servicer.export_development_image, + request_deserializer=instances__pb2.ExportDevelopmentImageRequest.FromString, + response_serializer=instances__pb2.ExportDevelopmentImageResponse.SerializeToString, + ), } generic_handler = grpc.method_handlers_generic_handler( 'omnivirt.InstanceGrpcService', rpc_method_handlers) @@ -130,3 +162,37 @@ class InstanceGrpcService(object): instances__pb2.DeleteInstanceResponse.FromString, options, channel_credentials, insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def take_snapshot(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/omnivirt.InstanceGrpcService/take_snapshot', + instances__pb2.TakeSnapshotRequest.SerializeToString, + instances__pb2.TakeSnapshotResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) + + @staticmethod + def export_development_image(request, + target, + options=(), + channel_credentials=None, + call_credentials=None, + insecure=False, + compression=None, + wait_for_ready=None, + timeout=None, + metadata=None): + return grpc.experimental.unary_unary(request, target, '/omnivirt.InstanceGrpcService/export_development_image', + instances__pb2.ExportDevelopmentImageRequest.SerializeToString, + instances__pb2.ExportDevelopmentImageResponse.FromString, + options, channel_credentials, + insecure, call_credentials, compression, wait_for_ready, timeout, metadata) diff --git a/eulerlauncher/grpcs/instances.py b/eulerlauncher/grpcs/instances.py index ec6bc29..797edbb 100644 --- a/eulerlauncher/grpcs/instances.py +++ b/eulerlauncher/grpcs/instances.py @@ -21,3 +21,15 @@ class Instance(object): request = instances_pb2.DeleteInstanceRequest(name=name) response = self.client.delete_instance(request) return response + + def take_snapshot(self, vm_name, snapshot_name, export_path): + """Take snapshot""" + request = instances_pb2.TakeSnapshotRequest(name=vm_name, snapshot=snapshot_name, dest_path=export_path) + response = self.client.take_snapshot(request) + return response + + def export_development_image(self, vm_name, image_name, export_path, pwd): + """Export Python/Go/Java development image""" + request = instances_pb2.ExportDevelopmentImageRequest(name=vm_name, image=image_name, dest_path=export_path, pwd=pwd) + response = self.client.export_development_image(request) + return response \ No newline at end of file diff --git a/eulerlauncher/services/instance_service.py b/eulerlauncher/services/instance_service.py index bea002e..46463f6 100644 --- a/eulerlauncher/services/instance_service.py +++ b/eulerlauncher/services/instance_service.py @@ -65,7 +65,7 @@ class InstanceService(instances_pb2_grpc.InstanceGrpcServiceServicer): return instances_pb2.CreateInstanceResponse(ret=2, msg=msg) vm = self.backend.create_instance( - request.name, request.image, self.instance_record_file, all_instances, all_img) + request.name, request.image, self.instance_record_file, all_instances, all_img, False, 0, 0) msg = f'Successfully created {request.name} with image {request.image}' return instances_pb2.CreateInstanceResponse(ret=1, msg=msg, instance=vm) @@ -79,3 +79,31 @@ class InstanceService(instances_pb2_grpc.InstanceGrpcServiceServicer): self.backend.delete_instance(request.name, self.instance_record_file, all_instances) msg = f'Successfully deleted instance: {request.name}.' return instances_pb2.DeleteInstanceResponse(ret=1, msg=msg) + + def take_snapshot(self, request, context): + LOG.debug(f"Get request to take snapshot: {request.name} with snapshot name {request.snapshot}, export to path: {request.dest_path}...") + all_img = utils.load_json_data(self.img_record_file) + all_instances = utils.load_json_data(self.instance_record_file) + if request.name not in all_instances['instances'].keys(): + msg = f'Error: Instance with name {request.name} does not exist.' + return instances_pb2.TakeSnapshotResponse(msg=msg) + export_path = self.backend.take_snapshot(request.name, request.snapshot, request.dest_path, all_instances, all_img, self.instance_record_file) + msg = f'Successfully take snapshot: {request.snapshot} of vm: {request.name} and export snapshot image to {export_path}.' + return instances_pb2.TakeSnapshotResponse(msg=msg) + + def export_development_image(self, request, context): + LOG.debug(f"Get request to export Python/Go/Java development image for: {request.name} with image name {request.image}, export to path: {request.dest_path}...") + all_img = utils.load_json_data(self.img_record_file) + all_instances = utils.load_json_data(self.instance_record_file) + if request.name not in all_instances['instances'].keys(): + msg = f'Error: Instance with name {request.name} does not exist.' + return instances_pb2.ExportDevelopmentImageResponse(msg=msg) + result = self.backend.make_development_image(request.name, request.pwd) + if result != 0: + LOG.debug(f"install Python/Go/Java environment for vm {request.name} failed!") + msg = f"install Python/Go/Java environment for vm {request.name} failed!" + return instances_pb2.ExportDevelopmentImageResponse(msg=msg) + LOG.debug(f"vm: {request.name} install Python/Go/Java development environment finished") + export_path = self.backend.take_snapshot(request.name, request.image, request.dest_path, all_instances, all_img, self.instance_record_file) + msg = f'Successfully export Python/Go/Java development image: {request.image} of vm: {request.name} and export development image to {export_path}.' + return instances_pb2.ExportDevelopmentImageResponse(msg=msg) \ No newline at end of file diff --git a/requirements-win.txt b/requirements-win.txt index f530041..7ed4143 100644 --- a/requirements-win.txt +++ b/requirements-win.txt @@ -14,6 +14,7 @@ oslo.i18n oslo.log oslo.serialization oslo.utils +paramiko Pillow prettytable protobuf @@ -22,4 +23,4 @@ pystray PyYAML six urllib3 -wget +wget \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 297a362..e29252c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,6 +15,7 @@ oslo.i18n oslo.log oslo.serialization oslo.utils +paramiko Pillow prettytable protobuf @@ -23,4 +24,4 @@ pystray PyYAML six urllib3 -wget +wget \ No newline at end of file -- Gitee From 6a1f960407698cbe533e25ccbbf0182557fa6b80 Mon Sep 17 00:00:00 2001 From: Chen Zheng <1072122585@qq.com> Date: Mon, 9 Oct 2023 19:32:17 +0800 Subject: [PATCH 4/5] update user manual document --- docs/mac-user-manual.md | 43 +++++++++++++++++++++++++--------------- docs/win-user-manual.md | 44 ++++++++++++++++++++++++++--------------- 2 files changed, 55 insertions(+), 32 deletions(-) diff --git a/docs/mac-user-manual.md b/docs/mac-user-manual.md index a2c0698..2e7b82e 100644 --- a/docs/mac-user-manual.md +++ b/docs/mac-user-manual.md @@ -133,18 +133,18 @@ eulerlauncher download-image 22.03-LTS Downloading: 22.03-LTS, this might take a while, please check image status with "images" command. ``` -镜像下载请求是一个异步请求,具体的下载动作将在后台完成,具体耗时与你的网络情况相关,整体的镜像下载流程包括下载、解压缩、格式转换等相关子流程,在下载过程中可以通过 `image` 命令随时查看下载进展与镜像状态: +镜像下载请求是一个异步请求,具体的下载动作将在后台完成,具体耗时与你的网络情况相关,整体的镜像下载流程包括下载、解压缩、格式转换等相关子流程,在下载过程中可以通过 `image` 命令随时查看下载进展与镜像状态,进度条格式为`([downloaded_bytes] [percentage] [download_speed] [remaining_download_time])`: ```Shell eulerlauncher images -+-----------+----------+--------------+ -| Images | Location | Status | -+-----------+----------+--------------+ -| 22.03-LTS | Remote | Downloadable | -| 21.09 | Remote | Downloadable | -| 22.03-LTS | Local | Downloading | -+-----------+----------+--------------+ ++-----------+----------+------------------------------------+ +| Images | Location | Status | ++-----------+----------+------------------------------------+ +| 22.03-LTS | Remote | Downloadable | +| 21.09 | Remote | Downloadable | +| 22.03-LTS | Local | Downloading: 33792K 8% 4.88M 55s | ++-----------+----------+------------------------------------+ ``` @@ -170,7 +170,7 @@ eulerlauncher images eulerlauncher load-image --path {image_file_path} IMAGE_NAME ``` -当前支持加载的镜像格式有 `xxx.qcow2.xz`,`xxx.qcow2` +当前支持加载的镜像格式有 `xxx.{qcow2, raw, vmdk, vhd, vhdx, qcow, vdi}.[xz]` 例如: @@ -185,13 +185,13 @@ Loading: 2203-load, this might take a while, please check image status with "ima ```Shell eulerlauncher images -+-----------+----------+--------------+ -| Images | Location | Status | -+-----------+----------+--------------+ -| 22.03-LTS | Remote | Downloadable | -| 21.09 | Remote | Downloadable | -| 2203-load | Local | Loading | -+-----------+----------+--------------+ ++-----------+----------+----------------------------+ +| Images | Location | Status | ++-----------+----------+----------------------------+ +| 22.03-LTS | Remote | Downloadable | +| 21.09 | Remote | Downloadable | +| 2203-load | Local | Loading: (24.00/100%) | ++-----------+----------+----------------------------+ eulerlauncher images @@ -255,6 +255,17 @@ eulerlauncher delete-instance {instance_name} ``` 根据虚拟机名称删除指定的虚拟机。 +5. 为虚拟机打快照,并导出为镜像 +```Shell +eulerlauncher take-snapshot --snapshot_name snap --export_path path vm_name +``` +通过`--snapshot_name`指定快照名称,`--export_path`指定导出镜像存放位置,`vm_name`为虚拟机名。 + +6. 将虚拟机导出为主流编程框架开发镜像 +```Shell +eulerlauncher export-development-image --image_name image --export_path path vm_name +``` +通过`--image_name`指定导出镜像名称,`--export_path`指定导出镜像存放位置,`vm_name`为虚拟机名,默认导出为Python/Go/Java主流编程框架开发镜像。 [1]: https://developer.apple.com/documentation/vmnet [2]: https://gitee.com/openeuler/eulerlauncher/releases \ No newline at end of file diff --git a/docs/win-user-manual.md b/docs/win-user-manual.md index 9744400..e7d69aa 100644 --- a/docs/win-user-manual.md +++ b/docs/win-user-manual.md @@ -60,18 +60,18 @@ eulerlauncher download-image 22.03-LTS Downloading: 22.03-LTS, this might take a while, please check image status with "images" command. ``` -镜像下载请求是一个异步请求,具体的下载动作将在后台完成,具体耗时与你的网络情况相关,整体的镜像下载流程包括下载、解压缩、格式转换等相关子流程,在下载过程中可以通过 `image` 命令随时查看下载进展与镜像状态: +镜像下载请求是一个异步请求,具体的下载动作将在后台完成,具体耗时与你的网络情况相关,整体的镜像下载流程包括下载、解压缩、格式转换等相关子流程,在下载过程中可以通过 `image` 命令随时查看下载进展与镜像状态,进度条格式为`[downloaded_bytes]/[total_bytes] (percentage)`: ```PowerShell eulerlauncher images -+-----------+----------+--------------+ -| Images | Location | Status | -+-----------+----------+--------------+ -| 22.03-LTS | Remote | Downloadable | -| 21.09 | Remote | Downloadable | -| 22.03-LTS | Local | Downloading | -+-----------+----------+--------------+ ++-----------+----------+------------------------------------+ +| Images | Location | Status | ++-----------+----------+------------------------------------+ +| 22.03-LTS | Remote | Downloadable | +| 21.09 | Remote | Downloadable | +| 22.03-LTS | Local | Downloading: 97.76/ 386.74MB (25%) | ++-----------+----------+------------------------------------+ ``` @@ -97,7 +97,7 @@ eulerlauncher images eulerlauncher load-image --path {image_file_path} IMAGE_NAME ``` -当前支持加载的镜像格式有 `xxx.qcow2.xz`,`xxx.qcow2` +当前支持加载的镜像格式有 `xxx.{qcow2, raw, vmdk, vhd, vhdx, qcow, vdi}.[xz]` 例如: @@ -112,13 +112,13 @@ Loading: 2203-load, this might take a while, please check image status with "ima ```PowerShell eulerlauncher images -+-----------+----------+--------------+ -| Images | Location | Status | -+-----------+----------+--------------+ -| 22.03-LTS | Remote | Downloadable | -| 21.09 | Remote | Downloadable | -| 2203-load | Local | Loading | -+-----------+----------+--------------+ ++-----------+----------+----------------------------+ +| Images | Location | Status | ++-----------+----------+----------------------------+ +| 22.03-LTS | Remote | Downloadable | +| 21.09 | Remote | Downloadable | +| 2203-load | Local | Loading: (24.00/100%) | ++-----------+----------+----------------------------+ eulerlauncher images @@ -182,5 +182,17 @@ eulerlauncher delete-instance {instance_name} ``` 根据虚拟机名称删除指定的虚拟机。 +5. 为虚拟机打快照,并导出为镜像 +```Shell +eulerlauncher take-snapshot --snapshot_name snap --export_path path vm_name +``` +通过`--snapshot_name`指定快照名称,`--export_path`指定导出镜像存放位置,`vm_name`为虚拟机名。 + +6. 将虚拟机导出为主流编程框架开发镜像 +```Shell +eulerlauncher export-development-image --image_name image --export_path path vm_name +``` +通过`--image_name`指定导出镜像名称,`--export_path`指定导出镜像存放位置,`vm_name`为虚拟机名,默认导出为Python/Go/Java主流编程框架开发镜像。 + [1]: https://gitee.com/openeuler/omnivirt/releases [2]: https://learn.microsoft.com/zh-cn/virtualization/hyper-v-on-windows/quick-start/enable-hyper-v \ No newline at end of file -- Gitee From 2fe9af9c435c74e481380660ba06456453bae21e Mon Sep 17 00:00:00 2001 From: Chen Zheng <1072122585@qq.com> Date: Wed, 18 Oct 2023 20:13:19 +0800 Subject: [PATCH 5/5] bugfix of previous merge --- eulerlauncher/backends/win/image_handler.py | 22 ++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/eulerlauncher/backends/win/image_handler.py b/eulerlauncher/backends/win/image_handler.py index 6fdc18e..f0398a5 100644 --- a/eulerlauncher/backends/win/image_handler.py +++ b/eulerlauncher/backends/win/image_handler.py @@ -139,14 +139,15 @@ class WinImageHandler(object): if self.pattern == 'hyper-v': # Convert the qcow2 img to vhdx vhdx_name = img_to_load + '.vhdx' - self.LOG.debug(f'Converting image file: {qcow2_name} to {vhdx_name} ...') - load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) - with powershell.PowerShell('GBK') as ps: - cmd = 'qemu-img convert -p -O vhdx {0} {1} | Out-File -FilePath {2}' - outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, vhdx_name), load_progress_bar_path)) - self.LOG.debug(f'Cleanup temp files ...') - os.remove(os.path.join(self.image_dir, qcow2_name)) - os.remove(load_progress_bar_path) + if fmt != 'vhdx': + self.LOG.debug(f'Converting image file: {qcow2_name} to {vhdx_name} ...') + load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) + with powershell.PowerShell('GBK') as ps: + cmd = 'qemu-img convert -p -O vhdx {0} {1} | Out-File -FilePath {2}' + outs, errs = ps.run(cmd.format(os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, vhdx_name), load_progress_bar_path)) + self.LOG.debug(f'Cleanup temp files ...') + os.remove(os.path.join(self.image_dir, qcow2_name)) + os.remove(load_progress_bar_path) image_name = vhdx_name elif self.pattern == 'qemu': image_name = qcow2_name @@ -156,7 +157,7 @@ class WinImageHandler(object): image.status = constants.IMAGE_STATUS_READY images['local'][image.name] = image.to_dict() omni_utils.save_json_data(self.image_record_file, images) - self.LOG.debug(f'Image: {vhdx_name} is ready ...') + self.LOG.debug(f'Image: {image_name} is ready ...') def load_progress_bar(self, img_name): progress_bar_lines = [] @@ -167,5 +168,4 @@ class WinImageHandler(object): if len(progress_bar_lines) > 1 and progress_bar_lines[-2].strip() != "": return constants.IMAGE_STATUS_LOADING + ": " + progress_bar_lines[-2].strip() else: - return constants.IMAGE_STATUS_LOADING - self.LOG.debug(f'Image: {image_name} is ready ...') \ No newline at end of file + return constants.IMAGE_STATUS_LOADING \ No newline at end of file -- Gitee