diff --git a/mindinsight/datavisual/common/log.py b/mindinsight/datavisual/common/log.py index 53fd669b2d3544cdbbd071cd0d1bf7fc8f9b7a0d..83394fa02c1687480fa7e639450be58715ce61a2 100644 --- a/mindinsight/datavisual/common/log.py +++ b/mindinsight/datavisual/common/log.py @@ -17,3 +17,5 @@ from mindinsight.utils.log import setup_logger logger = setup_logger("datavisual", "datavisual") restful_logger = setup_logger("restful_api", "restful_api") +parse_summary_logger = setup_logger("parse_summary", "parse_summary", console=True, + formatter='[%(levelname)s]%(message)s') diff --git a/mindinsight/datavisual/data_transform/summary_parser/__init__.py b/mindinsight/datavisual/data_transform/summary_parser/__init__.py new file mode 100644 index 0000000000000000000000000000000000000000..c0013c6883fc7337d624888a46feb7572d4645d2 --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_parser/__init__.py @@ -0,0 +1,15 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Summary parser module used to parse summary data and save it to csv file and image.""" diff --git a/mindinsight/datavisual/data_transform/summary_parser/event_parser.py b/mindinsight/datavisual/data_transform/summary_parser/event_parser.py new file mode 100644 index 0000000000000000000000000000000000000000..90ab80c4b928359f6de12fe9e2dd6364bc7a5736 --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_parser/event_parser.py @@ -0,0 +1,245 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Scalar Writer. + +This module write scalar into a csv file. +""" +import os +import struct + +from google.protobuf.message import DecodeError + +from mindinsight.datavisual.utils import crc32 +from mindinsight.datavisual.common import exceptions +from mindinsight.datavisual.common.log import parse_summary_logger +from mindinsight.datavisual.proto_files import lazy_read_pb2 +from mindinsight.datavisual.data_access.file_handler import FileHandler +from mindinsight.datavisual.data_transform.summary_parser.image_writer import ImageWriter +from mindinsight.datavisual.data_transform.summary_parser.scalar_writer import ScalarWriter +from mindinsight.datavisual.data_transform.ms_data_loader import _SummaryParser +from mindinsight.utils.exceptions import UnknownError + +HEADER_SIZE = 8 +CRC_STR_SIZE = 4 +MAX_EVENT_STRING = 500000000 +SCALAR = 'scalar_value' +IMAGE = 'image' + + +class EventParser(): + """Parse summary file and save it to csv file and image.""" + def __init__(self, summary_dir, output): + self._summary_dir = summary_dir + self._output = output + self._scalar_writer = ScalarWriter(self._output) + self._image_writer = ImageWriter(FileHandler.join(self._output, IMAGE)) + self._current = 0 + self._file_size = 0 + self._process_info = 0 + self._image_check = False + self._scalar_check = False + + def parse(self): + """Load summary file and parse file content.""" + try: + if not (self._check_filepath() and self._check_create_filepath( + self._output) and self._check_create_filepath(FileHandler.join(self._output, IMAGE))): + return + + summary_parser = _SummaryParser(self._summary_dir) + summary_files = summary_parser.filter_files(os.listdir(self._summary_dir)) + + if not summary_files: + parse_summary_logger.error('Path %s has no summary file.', self._summary_dir) + return + + summary_files = summary_parser.sort_files(summary_files) + + filename = summary_files[-1] + file_path = FileHandler.join(self._summary_dir, filename) + + if not os.access(file_path, os.R_OK): + parse_summary_logger.error('Path %s is not accessible, please check the file-authority.', file_path) + return + + self._summary_file_handler = FileHandler(file_path, 'rb') + + self._file_size = os.path.getsize(file_path) + # when current parsed size bigger than self._process_info, print process + self._process_info = self._file_size // 10 + + parse_summary_logger.info("loading %s", file_path) + result = self._load(self._summary_file_handler) + + self._scalar_writer.write() + + warning = '' + + if not self._scalar_check: + warning = warning + " the summary file contains no scalar value" + if not self._image_check: + warning = warning + " the summary file contains no image" + if result: + if warning: + parse_summary_logger.warning(warning) + parse_summary_logger.info("parsing summary file finished") + + except Exception as ex: + parse_summary_logger.error("Parse summary file failed, detail: %r", str(ex)) + raise UnknownError(str(ex)) + + def _load(self, file_handler): + """ + Load a log file data. + + Args: + file_handler (FileHandler): A file handler. + + Returns: + bool, True if the summary file is finished loading. + """ + while True: + try: + event_str = self._event_load(file_handler) + if event_str is None: + return True + if len(event_str) > MAX_EVENT_STRING: + parse_summary_logger.warning("file_path: %s, event string: %d exceeds %d and drop it.", + file_handler.file_path, len(event_str), MAX_EVENT_STRING) + continue + self._event_parse(event_str) + except exceptions.CRCFailedError: + parse_summary_logger.error("Check crc faild, file_path=%s, offset=%s.", file_handler.file_path, + file_handler.offset) + return False + except (OSError, DecodeError, exceptions.MindInsightException) as ex: + parse_summary_logger.error("Parse file fail, detail: %r, file path: %s.", str(ex), + file_handler.file_path) + return False + + def _event_load(self, file_handler): + """ + Load binary string to event string. + + Args: + file_handler (FileHandler): A file handler. + + Returns: + bytes, MindSpore event in bytes. + """ + # read the header + header_str = file_handler.read(HEADER_SIZE) + + if not header_str: + return None + + header_crc_str = file_handler.read(CRC_STR_SIZE) + if not header_crc_str: + header_crc_str = '' + + if len(header_str) != HEADER_SIZE or len(header_crc_str) != CRC_STR_SIZE: + parse_summary_logger.error("Check header size and crc, record truncated at offset %s, file_path=%s.", + file_handler.offset, file_handler.file_path) + return None + if not crc32.CheckValueAgainstData(header_crc_str, header_str, HEADER_SIZE): + raise exceptions.CRCFailedError() + + # read the event body if integrity of header is verified + header = struct.unpack('Q', header_str) + event_len = int(header[0]) + + event_str = file_handler.read(event_len) + if not event_str: + event_str = '' + event_crc_str = file_handler.read(CRC_STR_SIZE) + if not event_crc_str: + event_crc_str = '' + + if len(event_str) != event_len or len(event_crc_str) != CRC_STR_SIZE: + parse_summary_logger.error("Check event crc, record truncated at offset %d, file_path: %s.", + file_handler.offset, file_handler.file_path) + return None + if not crc32.CheckValueAgainstData(event_crc_str, event_str, event_len): + raise exceptions.CRCFailedError() + self._current += HEADER_SIZE + 2 * CRC_STR_SIZE + event_len + if self._current > self._process_info: + parse_summary_logger.info("current process: %d/%d, %d%%", self._current, self._file_size, + 100 * self._current // self._file_size) + self._process_info += self._file_size // 10 + return event_str + + def _event_parse(self, event_str): + """ + Transform `Event` data to event and extract the scalar and image data. + + Args: + event_str (str): Message event string in summary proto, data read from file handler. + """ + + plugins = [SCALAR, IMAGE] + + event = lazy_read_pb2.Event.FromString(event_str) + + if event.HasField('summary'): + for value in event.summary.value: + for plugin in plugins: + if not value.HasField(plugin): + continue + self._parse_summary_value(value.tag, event.step, event.wall_time, value, plugin) + + def _parse_summary_value(self, tag, step, wall_time, value, plugin): + """ + Parse summary value and write corresponding file according to plugin. + + Args: + tag (str): value tag + step (int): train step + wall_time (float): Timestamp + value (Summary.Value): Value message in summary file. + plugin (str): Plugin value. + """ + if plugin == SCALAR: + self._scalar_writer.add((tag, step, wall_time, value.scalar_value)) + self._scalar_check = True + + elif plugin == IMAGE: + self._image_writer.add((tag, step, value.image.encoded_image)) + self._image_writer.write() + self._image_check = True + + def _check_filepath(self): + """Check file path existence, accessible and available""" + if os.path.exists(self._summary_dir): + if not os.path.isdir(self._summary_dir): + parse_summary_logger.error('Path of summary directory is not a valid directory.') + return False + if not os.access(self._summary_dir, os.R_OK | os.X_OK): + parse_summary_logger.error('Path %s is not accessible, please check the file-authority.', + self._summary_dir) + return True + parse_summary_logger.error('Path of summary directory not exists.') + return False + + def _check_create_filepath(self, filepath): + """Check file path existence, accessible and available, if not exist create the file""" + permissions = os.R_OK | os.W_OK | os.X_OK + os.umask(permissions << 3 | permissions) + if os.path.exists(filepath): + parse_summary_logger.error('Path %s has already existed, please choose a new output path.', filepath) + return False + mode = permissions << 6 + os.makedirs(filepath, mode=mode) + return True diff --git a/mindinsight/datavisual/data_transform/summary_parser/image_writer.py b/mindinsight/datavisual/data_transform/summary_parser/image_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..435ff599d93f4bdccced20c0bd439d099e617a87 --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_parser/image_writer.py @@ -0,0 +1,56 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Image Writer. + +This module write scalar into a csv file. +""" +import os +import re + +from mindinsight.datavisual.data_transform.summary_parser.writer import Writer + + +class ImageWriter(Writer): + """ImageWriter write image into a png file.""" + def __init__(self, file_path): + """ + Init ImageWriter. + + Args: + file_path (str): A directory path, e.g. '/output/image/'. + """ + self._file_path = file_path + self._image_data = [] + + def add(self, value): + """ + Add value. + + Args: + value (object): tag and step and image value. + """ + self._image_data.append(value) + + def write(self): + """Write file.""" + for i in range(len(self._image_data)): + tag = self._image_data[i][0] + tag = tag.replace('/', '_') + tag = re.sub(r'[^a-zA-Z0-9_]+', '', tag) + with os.fdopen(os.open("{}/{}_{}.png".format(self._file_path, tag, self._image_data[i][1]), + os.O_WRONLY | os.O_CREAT, 0o600), 'wb') as fp: + fp.write(self._image_data[i][2]) + self._image_data = [] diff --git a/mindinsight/datavisual/data_transform/summary_parser/scalar_writer.py b/mindinsight/datavisual/data_transform/summary_parser/scalar_writer.py new file mode 100644 index 0000000000000000000000000000000000000000..6b70f9170f271e46d9a9a99fc5a0a60c51e6c0c2 --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_parser/scalar_writer.py @@ -0,0 +1,51 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +""" +Scalar Writer. + +This module write scalar into a csv file. +""" +import csv +import os + +from mindinsight.datavisual.data_transform.summary_parser.writer import Writer + + +class ScalarWriter(Writer): + """ScalarWriter write scalar into a csv file.""" + def __init__(self, file_path): + """ + Init ScalarWriter. + + Args: + file_path (str): A directory path, e.g. '/output/'. + """ + self._file_path = file_path + self._scalar_data = [("tag", "step", "wall_time", "value")] + + def add(self, value): + """ + Add value. + + Args: + value (object): wall_time, tag and step and scalar value. + """ + self._scalar_data.append(value) + + def write(self): + """Write file.""" + with os.fdopen(os.open('{}/scalar.csv'.format(self._file_path), os.O_WRONLY | os.O_CREAT, 0o600), 'w') as fp: + writer = csv.writer(fp, dialect='excel') + writer.writerows(self._scalar_data) diff --git a/mindinsight/datavisual/data_transform/summary_parser/writer.py b/mindinsight/datavisual/data_transform/summary_parser/writer.py new file mode 100644 index 0000000000000000000000000000000000000000..0cdcbef2f453606a5a4f31002493a45b4fb341a9 --- /dev/null +++ b/mindinsight/datavisual/data_transform/summary_parser/writer.py @@ -0,0 +1,32 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Base writer.""" +from abc import abstractmethod + + +class Writer: + """Base writer for writers.""" + @abstractmethod + def add(self, value): + """ + Abstract method for adding value. + + Args: + value (object): scalar, tensor or image value with wall_time, tag and step. + """ + + @abstractmethod + def write(self): + """Abstract method for writing file.""" diff --git a/mindinsight/datavisual/proto_files/lazy_read.proto b/mindinsight/datavisual/proto_files/lazy_read.proto new file mode 100644 index 0000000000000000000000000000000000000000..7bbb148cfef7e6a391804e9a18cd7a8d902c61d8 --- /dev/null +++ b/mindinsight/datavisual/proto_files/lazy_read.proto @@ -0,0 +1,45 @@ +syntax = "proto2"; + +package mindinsight.summary; +option cc_enable_arenas = true; + +// Event Protocol buffer, Top define +message Event { + // Timestamp + required double wall_time = 1; + + // The step of train. + optional int64 step = 2; + + oneof what { + // An event file was started, with the specified version. + // Now version is "Mindspore.Event:1" + string version = 3; + + // Summary data + Summary summary = 5; + + } +} + + +// A Summary is a set of named values that be produced regularly during training +message Summary { + message Image { + required bytes encoded_image = 4; + } + + message Value { + // Tag name for the data. + required string tag = 1; + + // Value associated with the tag. + oneof value { + float scalar_value = 3; + Image image = 4; + } + } + + // Set of values for the summary. + repeated Value value = 1; +} diff --git a/mindinsight/datavisual/proto_files/lazy_read_pb2.py b/mindinsight/datavisual/proto_files/lazy_read_pb2.py new file mode 100644 index 0000000000000000000000000000000000000000..20ed717634fc560cd2eb1a7dbf8ee02bf08b07a5 --- /dev/null +++ b/mindinsight/datavisual/proto_files/lazy_read_pb2.py @@ -0,0 +1,251 @@ +# -*- coding: utf-8 -*- +# Generated by the protocol buffer compiler. DO NOT EDIT! +# source: lazy_read.proto +"""Generated protocol buffer code.""" +from google.protobuf import descriptor as _descriptor +from google.protobuf import message as _message +from google.protobuf import reflection as _reflection +from google.protobuf import symbol_database as _symbol_database +# @@protoc_insertion_point(imports) + +_sym_db = _symbol_database.Default() + + + + +DESCRIPTOR = _descriptor.FileDescriptor( + name='lazy_read.proto', + package='mindinsight.summary', + syntax='proto2', + serialized_options=b'\370\001\001', + create_key=_descriptor._internal_create_key, + serialized_pb=b'\n\x0flazy_read.proto\x12\x13mindinsight.summary\"t\n\x05\x45vent\x12\x11\n\twall_time\x18\x01 \x02(\x01\x12\x0c\n\x04step\x18\x02 \x01(\x03\x12\x11\n\x07version\x18\x03 \x01(\tH\x00\x12/\n\x07summary\x18\x05 \x01(\x0b\x32\x1c.mindinsight.summary.SummaryH\x00\x42\x06\n\x04what\"\xc8\x01\n\x07Summary\x12\x31\n\x05value\x18\x01 \x03(\x0b\x32\".mindinsight.summary.Summary.Value\x1a\x1e\n\x05Image\x12\x15\n\rencoded_image\x18\x04 \x02(\x0c\x1aj\n\x05Value\x12\x0b\n\x03tag\x18\x01 \x02(\t\x12\x16\n\x0cscalar_value\x18\x03 \x01(\x02H\x00\x12\x33\n\x05image\x18\x04 \x01(\x0b\x32\".mindinsight.summary.Summary.ImageH\x00\x42\x07\n\x05valueB\x03\xf8\x01\x01' +) + + + + +_EVENT = _descriptor.Descriptor( + name='Event', + full_name='mindinsight.summary.Event', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='wall_time', full_name='mindinsight.summary.Event.wall_time', index=0, + number=1, type=1, cpp_type=5, label=2, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='step', full_name='mindinsight.summary.Event.step', index=1, + number=2, type=3, cpp_type=2, label=1, + has_default_value=False, default_value=0, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='version', full_name='mindinsight.summary.Event.version', index=2, + number=3, type=9, cpp_type=9, label=1, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='summary', full_name='mindinsight.summary.Event.summary', index=3, + number=5, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='what', full_name='mindinsight.summary.Event.what', + index=0, containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[]), + ], + serialized_start=40, + serialized_end=156, +) + + +_SUMMARY_IMAGE = _descriptor.Descriptor( + name='Image', + full_name='mindinsight.summary.Summary.Image', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='encoded_image', full_name='mindinsight.summary.Summary.Image.encoded_image', index=0, + number=4, type=12, cpp_type=9, label=2, + has_default_value=False, default_value=b"", + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=221, + serialized_end=251, +) + +_SUMMARY_VALUE = _descriptor.Descriptor( + name='Value', + full_name='mindinsight.summary.Summary.Value', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='tag', full_name='mindinsight.summary.Summary.Value.tag', index=0, + number=1, type=9, cpp_type=9, label=2, + has_default_value=False, default_value=b"".decode('utf-8'), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='scalar_value', full_name='mindinsight.summary.Summary.Value.scalar_value', index=1, + number=3, type=2, cpp_type=6, label=1, + has_default_value=False, default_value=float(0), + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + _descriptor.FieldDescriptor( + name='image', full_name='mindinsight.summary.Summary.Value.image', index=2, + number=4, type=11, cpp_type=10, label=1, + has_default_value=False, default_value=None, + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + _descriptor.OneofDescriptor( + name='value', full_name='mindinsight.summary.Summary.Value.value', + index=0, containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[]), + ], + serialized_start=253, + serialized_end=359, +) + +_SUMMARY = _descriptor.Descriptor( + name='Summary', + full_name='mindinsight.summary.Summary', + filename=None, + file=DESCRIPTOR, + containing_type=None, + create_key=_descriptor._internal_create_key, + fields=[ + _descriptor.FieldDescriptor( + name='value', full_name='mindinsight.summary.Summary.value', index=0, + number=1, type=11, cpp_type=10, label=3, + has_default_value=False, default_value=[], + message_type=None, enum_type=None, containing_type=None, + is_extension=False, extension_scope=None, + serialized_options=None, file=DESCRIPTOR, create_key=_descriptor._internal_create_key), + ], + extensions=[ + ], + nested_types=[_SUMMARY_IMAGE, _SUMMARY_VALUE, ], + enum_types=[ + ], + serialized_options=None, + is_extendable=False, + syntax='proto2', + extension_ranges=[], + oneofs=[ + ], + serialized_start=159, + serialized_end=359, +) + +_EVENT.fields_by_name['summary'].message_type = _SUMMARY +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['version']) +_EVENT.fields_by_name['version'].containing_oneof = _EVENT.oneofs_by_name['what'] +_EVENT.oneofs_by_name['what'].fields.append( + _EVENT.fields_by_name['summary']) +_EVENT.fields_by_name['summary'].containing_oneof = _EVENT.oneofs_by_name['what'] +_SUMMARY_IMAGE.containing_type = _SUMMARY +_SUMMARY_VALUE.fields_by_name['image'].message_type = _SUMMARY_IMAGE +_SUMMARY_VALUE.containing_type = _SUMMARY +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['scalar_value']) +_SUMMARY_VALUE.fields_by_name['scalar_value'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY_VALUE.oneofs_by_name['value'].fields.append( + _SUMMARY_VALUE.fields_by_name['image']) +_SUMMARY_VALUE.fields_by_name['image'].containing_oneof = _SUMMARY_VALUE.oneofs_by_name['value'] +_SUMMARY.fields_by_name['value'].message_type = _SUMMARY_VALUE +DESCRIPTOR.message_types_by_name['Event'] = _EVENT +DESCRIPTOR.message_types_by_name['Summary'] = _SUMMARY +_sym_db.RegisterFileDescriptor(DESCRIPTOR) + +Event = _reflection.GeneratedProtocolMessageType('Event', (_message.Message,), { + 'DESCRIPTOR' : _EVENT, + '__module__' : 'lazy_read_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.summary.Event) + }) +_sym_db.RegisterMessage(Event) + +Summary = _reflection.GeneratedProtocolMessageType('Summary', (_message.Message,), { + + 'Image' : _reflection.GeneratedProtocolMessageType('Image', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_IMAGE, + '__module__' : 'lazy_read_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary.Image) + }) + , + + 'Value' : _reflection.GeneratedProtocolMessageType('Value', (_message.Message,), { + 'DESCRIPTOR' : _SUMMARY_VALUE, + '__module__' : 'lazy_read_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary.Value) + }) + , + 'DESCRIPTOR' : _SUMMARY, + '__module__' : 'lazy_read_pb2' + # @@protoc_insertion_point(class_scope:mindinsight.summary.Summary) + }) +_sym_db.RegisterMessage(Summary) +_sym_db.RegisterMessage(Summary.Image) +_sym_db.RegisterMessage(Summary.Value) + + +DESCRIPTOR._options = None +# @@protoc_insertion_point(module_scope) diff --git a/mindinsight/scripts/parse_summary.py b/mindinsight/scripts/parse_summary.py new file mode 100644 index 0000000000000000000000000000000000000000..54451575e2a077659c354230a9c4cde76702d02f --- /dev/null +++ b/mindinsight/scripts/parse_summary.py @@ -0,0 +1,116 @@ +# Copyright 2020 Huawei Technologies Co., Ltd +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ +"""Parse summary file.""" + +import argparse +import os +import datetime + +from mindinsight.utils.command import BaseCommand +from mindinsight.datavisual.data_transform.summary_parser.event_parser import EventParser + + +class FilepathAction(argparse.Action): + """File directory action class definition.""" + + def __call__(self, parser_in, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser_in (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + summary_dir = values + if summary_dir.startswith('~'): + summary_dir = os.path.realpath(os.path.expanduser(summary_dir)) + + if not summary_dir.startswith('/'): + summary_dir = os.path.realpath(os.path.join(os.getcwd(), summary_dir)) + + summary_dir = os.path.realpath(summary_dir) + setattr(namespace, self.dest, summary_dir) + + +class OutputDirAction(argparse.Action): + """File directory action class definition.""" + + def __call__(self, parser_in, namespace, values, option_string=None): + """ + Inherited __call__ method from argparse.Action. + + Args: + parser_in (ArgumentParser): Passed-in argument parser. + namespace (Namespace): Namespace object to hold arguments. + values (object): Argument values with type depending on argument definition. + option_string (str): Optional string for specific argument name. Default: None. + """ + output = values + if output.startswith('~'): + output = os.path.realpath(os.path.expanduser(output)) + + if not output.startswith('/'): + output = os.path.realpath(os.path.join(os.getcwd(), output)) + + output = os.path.realpath(output) + setattr(namespace, self.dest, output) + + +class Command(BaseCommand): + """Start mindinsight service.""" + + name = 'parse_summary' + description = 'Parse summary file' + + def add_arguments(self, parser): + """ + Add arguments to parser. + + Args: + parser (ArgumentParser): Specify parser to which arguments are added. + """ + parser.add_argument( + '--summary-dir', + type=str, + action=FilepathAction, + default=os.path.realpath(os.getcwd()), + help=""" + Optional, specify path for summary file directory. + Default directory is the current working directory. + """) + + parser.add_argument( + '--output', + type=str, + action=OutputDirAction, + default=os.path.realpath(os.getcwd()), + help=""" + Optional, specify path for converted file directory. Default output + directory is `output` folder in the current working directory. + """) + + def run(self, args): + """ + Execute for start command. + + Args: + args (Namespace): Parsed arguments to hold customized parameters. + """ + date_time = datetime.datetime.now().strftime('output_%Y%m%d_%H%M%S_%f') + date_time = os.path.join(args.output, date_time) + eventparser = EventParser(args.summary_dir, date_time) + eventparser.parse()