diff --git a/eulerlauncher/backends/mac/image_handler.py b/eulerlauncher/backends/mac/image_handler.py index 122ac766a76348fd30d806c1017c3ef6355af89d..ffb6ef37e09b46a29699c4338bc7de9d1dc8ce45 100644 --- a/eulerlauncher/backends/mac/image_handler.py +++ b/eulerlauncher/backends/mac/image_handler.py @@ -1,4 +1,3 @@ -import copy import lzma import wget import os @@ -7,150 +6,114 @@ import shutil import ssl from eulerlauncher.utils import constants -from eulerlauncher.utils import utils as omni_utils -from eulerlauncher.utils import objs - +from eulerlauncher.utils import utils ssl._create_default_https_context = ssl._create_unverified_context class MacImageHandler(object): - def __init__(self, conf, work_dir, image_dir, image_record_file, - logger, base_dir) -> None: - self.conf = conf + def __init__(self, CONF, work_dir, image_dir, LOG) -> None: + self.conf = CONF self.work_dir = work_dir self.image_dir = image_dir - self.image_record_file = image_record_file - self.base_dir = base_dir - self.wget_bin = conf.conf.get('default', 'wget_dir') - self.qemu_img_bin = conf.conf.get('default', 'qemu_img_dir') - self.LOG = logger - - - def download_and_transform(self, images, img_to_download): - - # Download the image - img_name = wget.filename_from_url(images['remote'][img_to_download]['path']) - img_dict = copy.deepcopy(images['remote'][img_to_download]) - - if not os.path.exists(os.path.join(self.image_dir, img_name)): - self.LOG.debug(f'Downloading image: {img_to_download} from remote repo ...') - img_dict['location'] = constants.IMAGE_LOCATION_LOCAL - img_dict['status'] = constants.IMAGE_STATUS_DOWNLOADING - images['local'][img_to_download] = img_dict - omni_utils.save_json_data(self.image_record_file, images) - - download_progress_bar_path = os.path.join(self.image_dir, 'download_progress_bar_' + img_to_download) - download_cmd = [self.wget_bin, images['remote'][img_to_download]['path'], - '-O', os.path.join(self.image_dir, img_name), '--no-check-certificate', - '--progress=dot:mega', '-q', '--show-progress', f'-o {download_progress_bar_path}'] - self.LOG.debug(' '.join(download_cmd)) - subprocess.call(' '.join(download_cmd), shell=True) - #wget.download(url=images['remote'][img_to_download]['path'], out=os.path.join(self.image_dir, img_name), bar=None) - self.LOG.debug(f'Image: {img_to_download} succesfully downloaded from remote repo ...') + self.image_record_file = os.path.join(image_dir, 'images.json') + self.LOG = LOG - # Decompress the image - self.LOG.debug(f'Decompressing image file: {img_name} ...') - qcow2_name = img_name[:-3] - with open(os.path.join(self.image_dir, img_name), 'rb') as pr, open(os.path.join(self.image_dir, qcow2_name), 'wb') as pw: - data = pr.read() - data_dec = lzma.decompress(data) - pw.write(data_dec) - - self.LOG.debug(f'Cleanup temp files ...') - os.remove(os.path.join(self.image_dir, img_name)) - - # Record local image - if os.path.exists(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)): - os.remove(os.path.join(self.image_dir, "download_progress_bar_" + img_to_download)) - img_dict['status'] = constants.IMAGE_STATUS_READY - img_dict['path'] = os.path.join(self.image_dir, qcow2_name) - images['local'][img_to_download] = img_dict - omni_utils.save_json_data(self.image_record_file, images) - self.LOG.debug(f'Image: {img_to_download} is ready ...') - - def download_progress_bar(self, img_name): - progress_bar = [] - progress_bar_path = os.path.join(self.image_dir, 'download_progress_bar_' + img_name) - if os.path.exists(progress_bar_path): - with open(progress_bar_path, "r", encoding=omni_utils.detect_encoding(progress_bar_path), errors='ignore') as progress_bar_file: - progress_bar = progress_bar_file.readlines() - if len(progress_bar) > 1 and progress_bar[-2].strip() != "": - return constants.IMAGE_STATUS_DOWNLOADING + ": " + progress_bar[-2].strip() - else: - return constants.IMAGE_STATUS_DOWNLOADING - - def delete_image(self, images, img_to_delete): - if img_to_delete not in images['local'].keys(): + def list_images(self): + image_record = utils.load_json_data(self.image_record_file) + all_images = list(image_record["remote"].values()) + list(image_record["local"].values()) + return all_images + + def download_image(self, name): + image_record = utils.load_json_data(self.image_record_file) + if name not in image_record['remote'].keys(): + self.LOG.debug(f'Image: {name} not valid for download') return 1 - else: - return self._delete_image(images, img_to_delete) - - def _delete_image(self, images, img_to_delete): - img_path = images['local'][img_to_delete]['path'] - # TODO: Raise error message if image file not exists - if os.path.exists(img_path): - self.LOG.debug(f'Deleting: {img_path} ...') - os.remove(img_path) - - self.LOG.debug(f'Deleting: {img_to_delete} from image database ...') - del images['local'][img_to_delete] - omni_utils.save_json_data(self.image_record_file, images) - - return 0 - - def load_and_transform(self, images, img_to_load, path, fmt, update=False): - - if update: - self._delete_image(images, img_to_load) - image = objs.Image() - image.name = img_to_load - image.path = '' - image.location = constants.IMAGE_LOCATION_LOCAL - image.status = constants.IMAGE_STATUS_LOADING - images['local'][image.name] = image.to_dict() - omni_utils.save_json_data(self.image_record_file, images) + @utils.asyncwrapper + def download_and_transform(name): + image_record = utils.load_json_data(self.image_record_file) + image_url = image_record['remote'][name]['path'] + image_file = wget.filename_from_url(image_url) + + # Download the image + self.LOG.debug(f'Downloading image: {name} from remote repo ...') + image_record['local'][name] = { + 'name': name, + 'location': constants.IMAGE_LOCATION_LOCAL, + 'status': constants.IMAGE_STATUS_DOWNLOADING, + 'path': image_url + } + utils.save_json_data(self.image_record_file, image_record) + wget_bin = self.conf.conf.get('default', 'wget_dir') + download_cmd = [wget_bin, image_url, + '-O', os.path.join(self.image_dir, image_file), + '--no-check-certificate', + '--user-agent', 'Mozilla'] + self.LOG.debug(' '.join(download_cmd)) + subprocess.call(' '.join(download_cmd), shell=True) + self.LOG.debug(f'Image: {name} succesfully downloaded from remote repo ...') - if fmt not in constants.IMAGE_LOAD_SUPPORTED_TYPES_COMPRESSED: - img_name = f'{img_to_load}.{fmt}' - shutil.copyfile(path, os.path.join(self.image_dir, img_name)) - else: # Decompress the image - self.LOG.debug(f'Decompressing image file: {path} ...') - fmt = fmt.split('.')[0] - img_name = f'{img_to_load}.{fmt}' - with open(path, 'rb') as pr, open(os.path.join(self.image_dir, img_name), 'wb') as pw: + self.LOG.debug(f'Decompressing image: {image_file} ...') + with open(os.path.join(self.image_dir, image_file), 'rb') as pr, open(os.path.join(self.image_dir, name), 'wb') as pw: data = pr.read() data_dec = lzma.decompress(data) pw.write(data_dec) - - # Convert the img to qcow2 - qcow2_name = img_to_load + '.qcow2' - if fmt != "qcow2": - self.LOG.debug(f'Converting image file: {img_name} to {qcow2_name} ...') - load_progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_to_load) - cmd = 'sudo {0} convert -p -O qcow2 {1} {2} > {3}' - subprocess.call(cmd.format(self.qemu_img_bin, os.path.join(self.image_dir, img_name), os.path.join(self.image_dir, qcow2_name), load_progress_bar_path), shell=True) + self.LOG.debug(f'Cleanup temp files ...') - os.remove(os.path.join(self.image_dir, img_name)) - os.remove(load_progress_bar_path) - - # Record local image - image.path = os.path.join(self.image_dir, qcow2_name) - image.status = constants.IMAGE_STATUS_READY - images['local'][image.name] = image.to_dict() - omni_utils.save_json_data(self.image_record_file, images) - self.LOG.debug(f'Image: {qcow2_name} is ready ...') + os.remove(os.path.join(self.image_dir, image_file)) + + # Record local image + image_record = utils.load_json_data(self.image_record_file) + image_record['local'][name]['status'] = constants.IMAGE_STATUS_READY + image_record['local'][name]['path'] = os.path.join(self.image_dir, name) + utils.save_json_data(self.image_record_file, image_record) + self.LOG.debug(f'Image: {name} is ready ...') + + download_and_transform(name) + return 0 - def load_progress_bar(self, img_name): - progress_bar_lines = [] - progress_bar_path = os.path.join(self.image_dir, "load_progress_bar_" + img_name) - if os.path.exists(progress_bar_path): - with open(progress_bar_path, 'r', encoding=omni_utils.detect_encoding(progress_bar_path), errors='ignore') as progress_bar_file: - progress_bar_lines = progress_bar_file.readlines() - if len(progress_bar_lines) > 1 and progress_bar_lines[-2].strip() != "": - return constants.IMAGE_STATUS_LOADING + ": " + progress_bar_lines[-2].strip() - else: - return constants.IMAGE_STATUS_LOADING \ No newline at end of file + def load_image(self, name, path): + if not os.path.exists(path): + self.LOG.debug(f'Image: {path} does not exist') + return 1 + + supported, fmt = utils.check_file_tail(path, constants.IMAGE_LOAD_SUPPORTED_TYPES) + if not supported: + self.LOG.debug(f'Image: {name} not valid for load') + return 2 + + @utils.asyncwrapper + def load_and_transform(name, path): + image_record = utils.load_json_data(self.image_record_file) + supported, fmt = utils.check_file_tail(path, constants.IMAGE_LOAD_SUPPORTED_TYPES) + self.LOG.debug(f'Loading image: {name} from image file: {path} ...') + image_record['local'][name] = { + 'name': name, + 'location': constants.IMAGE_LOCATION_LOCAL, + 'status': constants.IMAGE_STATUS_LOADING, + 'path': '' + } + utils.save_json_data(self.image_record_file, image_record) + + if fmt == 'qcow2': + shutil.copyfile(path, os.path.join(self.image_dir, name)) + else: + # Decompress the image + self.LOG.debug(f'Decompressing image file: {path} ...') + with open(path, 'rb') as pr, open(os.path.join(self.image_dir, name), 'wb') as pw: + data = pr.read() + data_dec = lzma.decompress(data) + pw.write(data_dec) + + # Record local image + image_record = utils.load_json_data(self.image_record_file) + image_record['local'][name]['status'] = constants.IMAGE_STATUS_READY + image_record['local'][name]['path'] = os.path.join(self.image_dir, name) + utils.save_json_data(self.image_record_file, image_record) + self.LOG.debug(f'Image: {name} is ready ...') + + load_and_transform(name, path) + return 0