diff --git a/mindformers/models/multi_modal/base_multi_modal_processor.py b/mindformers/models/multi_modal/base_multi_modal_processor.py index 6b73616656be2aa4081101977d55dc07a7784fc5..18a73d5b63beacbe2e25a24c2f175b541e57f669 100644 --- a/mindformers/models/multi_modal/base_multi_modal_processor.py +++ b/mindformers/models/multi_modal/base_multi_modal_processor.py @@ -16,6 +16,7 @@ """ BaseImageToTextProcessor """ +import os import copy from typing import Optional, Union, List, Dict @@ -342,30 +343,42 @@ class BaseXModalToTextProcessor(BaseProcessor): shm_name_save_path = "./shm_name.txt" output_columns = copy.deepcopy(self.modal_transform.model_transform_template.output_columns) - # move position_ids to end of list - if "position_ids" in output_columns: - index = output_columns.index("position_ids") - output_columns.pop(index) - output_columns.append("position_ids") - for column in output_columns: - if column == "input_ids": - continue - data = other_data.pop(column, None) - data = np.array(data).astype(np.float32) - if data.ndim == 1: - data = data[None, :] - - shm = create_shm(data.nbytes, shm_name_save_path) - shared_array = np.ndarray(data.shape, dtype=np.float32, buffer=shm.buf) - shared_array[:] = data - - shm_name = encode_shm_name_to_int64(shm.name) - shape_value = encode_shape_to_int64(data.shape) - input_ids[index] = shm_name - input_ids[index + 1] = shape_value - index += 2 - return input_ids + shm_objects = [] + try: + # move position_ids to end of list + if "position_ids" in output_columns: + index = output_columns.index("position_ids") + output_columns.pop(index) + output_columns.append("position_ids") + + for column in output_columns: + if column == "input_ids": + continue + data = other_data.pop(column, None) + data = np.array(data).astype(np.float32) + if data.ndim == 1: + data = data[None, :] + + shm = create_shm(data.nbytes, shm_name_save_path) + shm_objects.append(shm) + shared_array = np.ndarray(data.shape, dtype=np.float32, buffer=shm.buf) + shared_array[:] = data + + shm_name = encode_shm_name_to_int64(shm.name) + shape_value = encode_shape_to_int64(data.shape) + input_ids[index] = shm_name + input_ids[index + 1] = shape_value + index += 2 + return input_ids + finally: + # free resources + for shm in shm_objects: + shm.close() + if hasattr(shm, 'unlink'): + shm.unlink() + + os.unlink(shm_name_save_path) def tokenize(self, inputs: List[Dict[str, str]]): """only for mindie""" diff --git a/mindformers/models/multi_modal/shm_utils.py b/mindformers/models/multi_modal/shm_utils.py index 8886d529be0339a9997c6b377a15401b8272babb..8064a979fe406b202b8d6bea7b5f2d98cc5334b6 100644 --- a/mindformers/models/multi_modal/shm_utils.py +++ b/mindformers/models/multi_modal/shm_utils.py @@ -243,8 +243,8 @@ def get_data_from_shm(shm_name_value, shape_value, dtype=np.float32): try: shm_name = decode_shm_name_from_int64(shm_name_value) shape = decode_shape_from_int64(shape_value) - shm = shared_memory.SharedMemory(name=shm_name) - pixel_values = np.ndarray(shape, dtype=dtype, buffer=shm.buf).copy() + with shared_memory.SharedMemory(name=shm_name) as shm: + pixel_values = np.ndarray(shape, dtype=dtype, buffer=shm.buf).copy() + return pixel_values except Exception as e: raise ValueError(f"Get data from share memory error: {e}") from e - return pixel_values diff --git a/mindformers/parallel_core/training_graph/transformer/utils.py b/mindformers/parallel_core/training_graph/transformer/utils.py index 25f200832157a20e4725143d9f676a7dc13cd9b5..c44c100f7e564d5965d379e3b1ebd054ad34941b 100644 --- a/mindformers/parallel_core/training_graph/transformer/utils.py +++ b/mindformers/parallel_core/training_graph/transformer/utils.py @@ -15,7 +15,7 @@ """utils""" __all__ = ["get_attn_mask_func"] -import re +import regex import numpy as np import mindspore as ms from mindspore.ops import operations as P @@ -93,6 +93,14 @@ def get_attn_mask_func(mask_func_type): return ATTNMASK_FUNC_MAP[mask_func_type] +def regex_match(pattern, string, timeout=1): + try: + return regex.fullmatch(pattern, string, timeout=timeout) + except TimeoutError as e: + logger.warning(f"{e} Please check and fix it.") + return [] + + class LayerSetting: r""" Class for setting offset, pipeline stage, swap and select recompute for each transformer layer. @@ -218,7 +226,7 @@ class LayerSetting: if len(key) != len(split_pattern): continue for p1, p2 in zip(key, split_pattern): - if not re.fullmatch(p1, p2) and not re.fullmatch(p2, p1): + if not regex_match(p1, p2) and not regex_match(p2, p1): break else: repeat_key = pattern @@ -355,20 +363,20 @@ class LayerSetting: if p_list: # pylint: disable=W0212 for name, cell in layer._cells.items(): - if re.fullmatch(p, name): + if regex_match(p, name): log = LayerSetting.set_pattern_swap(cell, p_list, value, info + f'.{name}') if log: log_list.append(log[1:]) else: for attr in dir(layer): - if re.fullmatch(p, attr): + if regex_match(p, attr): operator = getattr(layer, attr) if hasattr(operator, '_offload'): operator._offload(backward_prefetch=value) log = f"{info}.{attr}, value={value}" # pylint: disable=W0212 for name, cell in layer._cells.items(): - if re.fullmatch(p, name): + if regex_match(p, name): cell.offload(backward_prefetch=value) log = f"{info}.{name}, value={value}" p_list.insert(0, p) @@ -584,13 +592,13 @@ class LayerSetting: if p_list: # pylint: disable=W0212 for name, cell in layer._cells.items(): - if re.fullmatch(p, name): + if regex_match(p, name): log = LayerSetting.set_pattern_recompute(cell, p_list, add_prim_attr, set_on, info + f'.{name}') if log: log_list.append(log[1:]) else: for attr in dir(layer): - if re.fullmatch(p, attr): + if regex_match(p, attr): operator = getattr(layer, attr) if add_prim_attr: operator.add_prim_attr("recompute_comm_op", set_on) @@ -600,7 +608,7 @@ class LayerSetting: log = f"{info}.{attr}" # pylint: disable=W0212 for name, cell in layer._cells.items(): - if re.fullmatch(p, name): + if regex_match(p, name): if not set_on: logger.info(f"For select recompute/comm_recompute exclude, {info.replace('.', '', 1)}.{name} " "is expected to be operation but got cell, " diff --git a/mindformers/tools/image_tools.py b/mindformers/tools/image_tools.py index 28a5b3f54c4f534fc6c004fe8f4a0216dcb20465..46d0108cc1766287704c5090ffaffdd982c6715e 100644 --- a/mindformers/tools/image_tools.py +++ b/mindformers/tools/image_tools.py @@ -33,12 +33,11 @@ def load_image(content, timeout=4): if content.startswith("https://") or content.startswith("http://"): try: - response = requests.get(content, stream=True, timeout=timeout) + with requests.get(content, stream=True, timeout=timeout) as response: + content = response.raw except (TimeoutError, urllib3.exceptions.MaxRetryError, requests.exceptions.ProxyError) as exc: raise ConnectionError(f"Connect error, please download {content}.") from exc - - content = response.raw elif not os.path.isfile(content): raise ValueError( f"{content} is not a valid path. If URL, it must start with `http://` or `https://`."