From 6787a8665e4c4d6c89fc0af53673f1421ed09c2b Mon Sep 17 00:00:00 2001 From: kurnevichstanislav Date: Tue, 13 Jun 2023 15:25:39 +0300 Subject: [PATCH] [Sampling profiler] Add options for building various flamegraphs * Add CLI args parser for aspt converter * Add 3 modes for aspt converter * Add thread_id in .aspt format Signed-off-by: kurnevichstanislav --- CMakeLists.txt | 2 +- plugins/ets/tests/napi/sampler/CMakeLists.txt | 2 +- .../runtime/tooling/sampler/CMakeLists.txt | 2 +- .../sampler/sampling_profiler_test.cpp | 17 +- runtime/tooling/sampler/sample_info.h | 73 ++++++-- runtime/tooling/sampler/sample_reader-inl.h | 22 +-- runtime/tooling/sampler/sample_reader.h | 12 +- runtime/tooling/sampler/sample_writer.cpp | 14 +- runtime/tooling/sampler/sample_writer.h | 11 +- runtime/tooling/sampler/sampling_profiler.cpp | 16 +- tools/sampler/CMakeLists.txt | 5 + tools/sampler/README.md | 11 +- tools/sampler/args_parser.h | 98 +++++++++++ tools/sampler/aspt_convert.cpp | 24 +-- tools/sampler/aspt_converter.h | 63 +++---- tools/sampler/options.yaml | 38 ++++ tools/sampler/trace_dumper.h | 165 ++++++++++++++++++ 17 files changed, 471 insertions(+), 104 deletions(-) create mode 100644 tools/sampler/args_parser.h create mode 100644 tools/sampler/options.yaml create mode 100644 tools/sampler/trace_dumper.h diff --git a/CMakeLists.txt b/CMakeLists.txt index 81e6ad36a..5e483df31 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -306,7 +306,7 @@ endif() # ----- Aliases ---------------------------------------------------------------- add_custom_target(panda_bins COMMENT "Build all common Panda binaries") -add_dependencies(panda_bins panda pandasm ark_disasm paoc verifier) +add_dependencies(panda_bins panda pandasm ark_disasm paoc verifier aspt_converter) add_dependencies(panda_gen_files panda_bins) # ----- Benchmarking ----------------------------------------------------------- diff --git a/plugins/ets/tests/napi/sampler/CMakeLists.txt b/plugins/ets/tests/napi/sampler/CMakeLists.txt index b48fd2f1b..7ddc56e2b 100644 --- a/plugins/ets/tests/napi/sampler/CMakeLists.txt +++ b/plugins/ets/tests/napi/sampler/CMakeLists.txt @@ -81,7 +81,7 @@ foreach(c_arg ${COMPILER_ARGUMENTS_LIST}) ) add_custom_target(sampler_etsnapi_run_aspt_convertor_${c_arg}_${i_arg} - COMMAND ${aspt_converter_bin} ${testing_dir}/stack_output_${c_arg}_${i_arg}.aspt ${testing_dir}/traceout_${c_arg}_${i_arg}.csv + COMMAND ${aspt_converter_bin} --input=${testing_dir}/stack_output_${c_arg}_${i_arg}.aspt --output=${testing_dir}/traceout_${c_arg}_${i_arg}.csv WORKING_DIRECTORY "${SAMPLER_GEN_ETSNAPI_LIB_DIR}" DEPENDS sampler_etsnapi_run_${c_arg}_${i_arg} ${aspt_converter_target} ) diff --git a/plugins/ets/tests/runtime/tooling/sampler/CMakeLists.txt b/plugins/ets/tests/runtime/tooling/sampler/CMakeLists.txt index 3dffcf7e3..f61c08520 100644 --- a/plugins/ets/tests/runtime/tooling/sampler/CMakeLists.txt +++ b/plugins/ets/tests/runtime/tooling/sampler/CMakeLists.txt @@ -77,7 +77,7 @@ foreach(c_arg ${COMPILER_ARGUMENTS_LIST}) ) add_custom_target(sampler_run_aspt_converter_${c_arg}_${i_arg} - COMMAND ${aspt_converter_bin} ${testing_dir}/stack_output_${c_arg}_${i_arg}.aspt ${testing_dir}/traceout_${c_arg}_${i_arg}.csv + COMMAND ${aspt_converter_bin} --input=${testing_dir}/stack_output_${c_arg}_${i_arg}.aspt --output=${testing_dir}/traceout_${c_arg}_${i_arg}.csv WORKING_DIRECTORY "${testing_dir}" DEPENDS sampler_run_${c_arg}_${i_arg} ${aspt_converter_target} ) diff --git a/runtime/tests/tooling/sampler/sampling_profiler_test.cpp b/runtime/tests/tooling/sampler/sampling_profiler_test.cpp index 996b1132c..682c81c17 100644 --- a/runtime/tests/tooling/sampler/sampling_profiler_test.cpp +++ b/runtime/tests/tooling/sampler/sampling_profiler_test.cpp @@ -72,10 +72,11 @@ public: void FullfillFakeSample(SampleInfo *ps) { - for (uint32_t i = 0; i < SampleInfo::MAX_STACK_DEPTH; ++i) { - ps->managed_stack[i] = {i, pf_id_}; + for (uint32_t i = 0; i < SampleInfo::StackInfo::MAX_STACK_DEPTH; ++i) { + ps->stack_info.managed_stack[i] = {i, pf_id_}; } - ps->managed_stack_size = SampleInfo::MAX_STACK_DEPTH; + ps->thread_info.thread_id = GetThreadId(); + ps->stack_info.managed_stack_size = SampleInfo::StackInfo::MAX_STACK_DEPTH; } // Friend wrappers for accesing samplers private fields @@ -119,6 +120,11 @@ public: return s_ptr->is_active_; } + uint32_t GetThreadId() + { + return os::thread::GetCurrentThreadId(); + } + protected: // NOLINTNEXTLINE(misc-non-private-member-variables-in-classes) panda::MTManagedThread *thread_ {nullptr}; @@ -724,11 +730,12 @@ TEST_F(SamplerTest, ProfilerSamplerSignalHandlerTest) while (reader.GetNextSample(&sample)) { ++sample_counter; - if (sample.managed_stack_size == 2) { + if (sample.stack_info.managed_stack_size == 2) { is_find = true; continue; } - ASSERT_NE(sample.managed_stack_size, 0); + ASSERT_NE(sample.stack_info.managed_stack_size, 0); + ASSERT_EQ(GetThreadId(), sample.thread_info.thread_id); } ASSERT_EQ(is_find, true); diff --git a/runtime/tooling/sampler/sample_info.h b/runtime/tooling/sampler/sample_info.h index bf38e4602..cae6cac81 100644 --- a/runtime/tooling/sampler/sample_info.h +++ b/runtime/tooling/sampler/sample_info.h @@ -21,6 +21,7 @@ #include #include "libpandabase/macros.h" +#include "libpandabase/os/thread.h" namespace panda::tooling::sampler { @@ -30,11 +31,22 @@ struct SampleInfo { uintptr_t file_id {0}; uintptr_t panda_file_ptr {0}; }; - static constexpr size_t MAX_STACK_DEPTH = 100; - // We can't use dynamic memory 'cause of usage inside the signal handler - std::array managed_stack; - uintptr_t managed_stack_size {0}; + struct StackInfo { + static constexpr size_t MAX_STACK_DEPTH = 100; + uintptr_t managed_stack_size {0}; + // We can't use dynamic memory 'cause of usage inside the signal handler + std::array managed_stack; + }; + + struct ThreadInfo { + // Id of the thread from which sample was obtained + uint32_t thread_id {0}; + // TODO(m.morozov): add info about thread status + }; + + ThreadInfo thread_info {}; + StackInfo stack_info {}; }; // Saving one module info (panda file, .so) @@ -49,6 +61,10 @@ bool operator==(const FileInfo &lhs, const FileInfo &rhs); bool operator!=(const FileInfo &lhs, const FileInfo &rhs); bool operator==(const SampleInfo::ManagedStackFrameId &lhs, const SampleInfo::ManagedStackFrameId &rhs); bool operator!=(const SampleInfo::ManagedStackFrameId &lhs, const SampleInfo::ManagedStackFrameId &rhs); +bool operator==(const SampleInfo::StackInfo &lhs, const SampleInfo::StackInfo &rhs); +bool operator!=(const SampleInfo::StackInfo &lhs, const SampleInfo::StackInfo &rhs); +bool operator==(const SampleInfo::ThreadInfo &lhs, const SampleInfo::ThreadInfo &rhs); +bool operator!=(const SampleInfo::ThreadInfo &lhs, const SampleInfo::ThreadInfo &rhs); inline uintptr_t ReadUintptrTBitMisaligned(const void *ptr) { @@ -56,9 +72,15 @@ inline uintptr_t ReadUintptrTBitMisaligned(const void *ptr) * Pointer might be misaligned * To avoid of UB we should read misaligned adresses with memcpy */ - std::array buf = {}; - memcpy(buf.data(), ptr, sizeof(uintptr_t)); - uintptr_t value = *reinterpret_cast(buf.data()); + uintptr_t value = 0; + memcpy(&value, ptr, sizeof(value)); + return value; +} + +inline uint32_t ReadUint32TBitMisaligned(const void *ptr) +{ + uint32_t value = 0; + memcpy(&value, ptr, sizeof(value)); return value; } @@ -82,18 +104,42 @@ inline bool operator!=(const FileInfo &lhs, const FileInfo &rhs) return !(lhs == rhs); } -inline bool operator==(const SampleInfo &lhs, const SampleInfo &rhs) +inline bool operator==(const SampleInfo::StackInfo &lhs, const SampleInfo::StackInfo &rhs) { if (lhs.managed_stack_size != rhs.managed_stack_size) { return false; } - for (uint32_t i = 0; i < lhs.managed_stack_size; ++i) { if (lhs.managed_stack[i] != rhs.managed_stack[i]) { return false; } } + return true; +} +inline bool operator!=(const SampleInfo::StackInfo &lhs, const SampleInfo::StackInfo &rhs) +{ + return !(lhs == rhs); +} + +inline bool operator==(const SampleInfo::ThreadInfo &lhs, const SampleInfo::ThreadInfo &rhs) +{ + return lhs.thread_id == rhs.thread_id; +} + +inline bool operator!=(const SampleInfo::ThreadInfo &lhs, const SampleInfo::ThreadInfo &rhs) +{ + return !(lhs == rhs); +} + +inline bool operator==(const SampleInfo &lhs, const SampleInfo &rhs) +{ + if (lhs.thread_info != rhs.thread_info) { + return false; + } + if (lhs.stack_info != rhs.stack_info) { + return false; + } return true; } @@ -111,12 +157,13 @@ template <> struct hash { std::size_t operator()(const panda::tooling::sampler::SampleInfo &s) const { - ASSERT(s.managed_stack_size <= panda::tooling::sampler::SampleInfo::MAX_STACK_DEPTH); + auto stack_info = s.stack_info; + ASSERT(stack_info.managed_stack_size <= panda::tooling::sampler::SampleInfo::StackInfo::MAX_STACK_DEPTH); size_t summ = 0; - for (size_t i = 0; i < s.managed_stack_size; ++i) { - summ += s.managed_stack[i].panda_file_ptr ^ s.managed_stack[i].file_id; + for (size_t i = 0; i < stack_info.managed_stack_size; ++i) { + summ += stack_info.managed_stack[i].panda_file_ptr ^ stack_info.managed_stack[i].file_id; } - return std::hash()(summ ^ s.managed_stack_size); + return std::hash()((summ ^ stack_info.managed_stack_size) + s.thread_info.thread_id); } }; diff --git a/runtime/tooling/sampler/sample_reader-inl.h b/runtime/tooling/sampler/sample_reader-inl.h index 2599f9a22..cd08ccd92 100644 --- a/runtime/tooling/sampler/sample_reader-inl.h +++ b/runtime/tooling/sampler/sample_reader-inl.h @@ -31,9 +31,9 @@ namespace panda::tooling::sampler { /* * Example for 64 bit architecture * - * Stack Size Stack frames - * Sample row |__________|_____________------___________| - * 64 bits (128 * ) bits + * Thread id Stack Size Managed stack frame id + * Sample row |_________|____________|_____________------___________| + * 32 bits 64 bits (128 * ) bits * * 0xFF..FF pointer name size module path (ASCII str) * Module row |__________|__________|__________|_____________------___________| @@ -76,9 +76,9 @@ inline SampleReader::SampleReader(const char *filename) continue; } - // This entry is a sample - size_t stack_size = ReadUintptrTBitMisaligned(&buffer_[buffer_counter]); - if (stack_size > SampleInfo::MAX_STACK_DEPTH) { + // buffer_counter now is entry of a sample, stack size lies after thread_id + size_t stack_size = ReadUintptrTBitMisaligned(&buffer_[buffer_counter + SAMPLE_STACK_SIZE_OFFSET]); + if (stack_size > SampleInfo::StackInfo::MAX_STACK_DEPTH) { LOG(FATAL, PROFILER) << "ark sampling profiler trace file is invalid, stack_size > MAX_STACK_DEPTH"; UNREACHABLE(); } @@ -105,11 +105,13 @@ inline bool SampleReader::GetNextSample(SampleInfo *sample_out) return false; } const char *current_sample_ptr = sample_row_ptrs_[sample_row_counter_]; - sample_out->managed_stack_size = ReadUintptrTBitMisaligned(¤t_sample_ptr[SAMPLE_STACK_SIZE_OFFSET]); + sample_out->thread_info.thread_id = ReadUint32TBitMisaligned(¤t_sample_ptr[SAMPLE_THREAD_ID_OFFSET]); + sample_out->stack_info.managed_stack_size = + ReadUintptrTBitMisaligned(¤t_sample_ptr[SAMPLE_STACK_SIZE_OFFSET]); - ASSERT(sample_out->managed_stack_size <= SampleInfo::MAX_STACK_DEPTH); - memcpy(sample_out->managed_stack.data(), current_sample_ptr + SAMPLE_STACK_OFFSET, - sample_out->managed_stack_size * sizeof(SampleInfo::ManagedStackFrameId)); + ASSERT(sample_out->stack_info.managed_stack_size <= SampleInfo::StackInfo::MAX_STACK_DEPTH); + memcpy(sample_out->stack_info.managed_stack.data(), current_sample_ptr + SAMPLE_STACK_OFFSET, + sample_out->stack_info.managed_stack_size * sizeof(SampleInfo::ManagedStackFrameId)); ++sample_row_counter_; return true; } diff --git a/runtime/tooling/sampler/sample_reader.h b/runtime/tooling/sampler/sample_reader.h index 28046ab12..ac7ae3c9f 100644 --- a/runtime/tooling/sampler/sample_reader.h +++ b/runtime/tooling/sampler/sample_reader.h @@ -25,12 +25,14 @@ namespace panda::tooling::sampler { // Reader of .aspt format class SampleReader final { public: - static constexpr size_t SAMPLE_STACK_SIZE_OFFSET = 0 * sizeof(uintptr_t); - static constexpr size_t SAMPLE_STACK_OFFSET = 1 * sizeof(uintptr_t); - static constexpr size_t PANDA_FILE_POINTER_OFFSET = 1 * sizeof(uintptr_t); + // clang-format off + static constexpr size_t SAMPLE_THREAD_ID_OFFSET = 0 * sizeof(uint32_t); + static constexpr size_t SAMPLE_STACK_SIZE_OFFSET = 1 * sizeof(uint32_t); + static constexpr size_t SAMPLE_STACK_OFFSET = 1 * sizeof(uint32_t) + 1 * sizeof(uintptr_t); + static constexpr size_t PANDA_FILE_POINTER_OFFSET = 1 * sizeof(uintptr_t); static constexpr size_t PANDA_FILE_NAME_SIZE_OFFSET = 2 * sizeof(uintptr_t); - static constexpr size_t PANDA_FILE_NAME_OFFSET = 3 * sizeof(uintptr_t); - + static constexpr size_t PANDA_FILE_NAME_OFFSET = 3 * sizeof(uintptr_t); + // clang-format on inline explicit SampleReader(const char *filename); ~SampleReader() = default; diff --git a/runtime/tooling/sampler/sample_writer.cpp b/runtime/tooling/sampler/sample_writer.cpp index 689a0a64c..fbe6fe67b 100644 --- a/runtime/tooling/sampler/sample_writer.cpp +++ b/runtime/tooling/sampler/sample_writer.cpp @@ -25,14 +25,16 @@ namespace panda::tooling::sampler { void StreamWriter::WriteSample(const SampleInfo &sample) const { ASSERT(write_stream_ptr_ != nullptr); - ASSERT(sample.managed_stack_size <= SampleInfo::MAX_STACK_DEPTH); + ASSERT(sample.stack_info.managed_stack_size <= SampleInfo::StackInfo::MAX_STACK_DEPTH); - static_assert(sizeof(sample.managed_stack_size) == sizeof(uintptr_t)); + static_assert(sizeof(sample.stack_info.managed_stack_size) == sizeof(uintptr_t)); - write_stream_ptr_->write(reinterpret_cast(&sample.managed_stack_size), - sizeof(sample.managed_stack_size)); - write_stream_ptr_->write(reinterpret_cast(sample.managed_stack.data()), - sample.managed_stack_size * sizeof(SampleInfo::ManagedStackFrameId)); + write_stream_ptr_->write(reinterpret_cast(&sample.thread_info.thread_id), + sizeof(sample.thread_info.thread_id)); + write_stream_ptr_->write(reinterpret_cast(&sample.stack_info.managed_stack_size), + sizeof(sample.stack_info.managed_stack_size)); + write_stream_ptr_->write(reinterpret_cast(sample.stack_info.managed_stack.data()), + sample.stack_info.managed_stack_size * sizeof(SampleInfo::ManagedStackFrameId)); } void StreamWriter::WriteModule(const FileInfo &module_info) const diff --git a/runtime/tooling/sampler/sample_writer.h b/runtime/tooling/sampler/sample_writer.h index a282ceede..89bd95824 100644 --- a/runtime/tooling/sampler/sample_writer.h +++ b/runtime/tooling/sampler/sample_writer.h @@ -46,16 +46,17 @@ namespace panda::tooling::sampler { * next 8 byte is size of panda file name * next bytes is panda file name in ASCII symbols * - * sample row: - * first 8 bytes is stack size + * sample row for 64-bits: + * first 4 bytes is thread id of thread from sample was obtained + * next 8 bytes is stack size * next bytes is stack frame * one stack frame is panda file ptr and file id * * Example for 64-bit architecture: * - * Stack Size Stack frames - * Sample row |__________|_____________------___________| - * 64 bits (128 * ) bits + * Thread id Stack Size Stack frames + * Sample row |_________|____________|_____________------___________| + * 32 bits 64 bits (128 * ) bits * * 0xFF..FF pointer name size module path (ASCII str) * Module row |__________|__________|__________|_____________------___________| diff --git a/runtime/tooling/sampler/sampling_profiler.cpp b/runtime/tooling/sampler/sampling_profiler.cpp index ec5fe327a..eb53ef308 100644 --- a/runtime/tooling/sampler/sampling_profiler.cpp +++ b/runtime/tooling/sampler/sampling_profiler.cpp @@ -370,13 +370,13 @@ void SigProfSamplingProfilerHandler([[maybe_unused]] int signum, [[maybe_unused] return; } - sample.managed_stack[stack_counter].panda_file_ptr = pf_id; - sample.managed_stack[stack_counter].file_id = method->GetFileId().GetOffset(); + sample.stack_info.managed_stack[stack_counter].panda_file_ptr = pf_id; + sample.stack_info.managed_stack[stack_counter].file_id = method->GetFileId().GetOffset(); ++stack_counter; stack_walker.NextFrame(); - if (stack_counter == SampleInfo::MAX_STACK_DEPTH) { + if (stack_counter == SampleInfo::StackInfo::MAX_STACK_DEPTH) { // According to the limitations we should drop all frames that is higher than MAX_STACK_DEPTH break; } @@ -384,7 +384,9 @@ void SigProfSamplingProfilerHandler([[maybe_unused]] int signum, [[maybe_unused] if (stack_counter == 0) { return; } - sample.managed_stack_size = stack_counter; + sample.stack_info.managed_stack_size = stack_counter; + sample.thread_info.thread_id = os::thread::GetCurrentThreadId(); + const ThreadCommunicator &communicator = Sampler::GetSampleCommunicator(); communicator.SendSample(sample); } @@ -431,7 +433,7 @@ void Sampler::SamplerThreadEntry() // Sending last sample on finish to avoid of deadlock in listener SampleInfo last_sample; - last_sample.managed_stack_size = 0; + last_sample.stack_info.managed_stack_size = 0; communicator_.SendSample(last_sample); --S_CURRENT_HANDLERS_COUNTER; @@ -455,7 +457,7 @@ void Sampler::ListenerThreadEntry(std::string output_file) while (is_active_.load(std::memory_order_relaxed)) { WriteLoadedPandaFiles(writer_ptr.get()); communicator_.ReadSample(&buffer_sample); - if (LIKELY(buffer_sample.managed_stack_size != 0)) { + if (LIKELY(buffer_sample.stack_info.managed_stack_size != 0)) { writer_ptr->WriteSample(buffer_sample); } } @@ -463,7 +465,7 @@ void Sampler::ListenerThreadEntry(std::string output_file) while (!communicator_.IsPipeEmpty()) { WriteLoadedPandaFiles(writer_ptr.get()); communicator_.ReadSample(&buffer_sample); - if (LIKELY(buffer_sample.managed_stack_size != 0)) { + if (LIKELY(buffer_sample.stack_info.managed_stack_size != 0)) { writer_ptr->WriteSample(buffer_sample); } } diff --git a/tools/sampler/CMakeLists.txt b/tools/sampler/CMakeLists.txt index f54a87559..ab2c9b407 100644 --- a/tools/sampler/CMakeLists.txt +++ b/tools/sampler/CMakeLists.txt @@ -21,4 +21,9 @@ target_link_libraries(aspt_converter arkbase ) +target_include_directories(aspt_converter PUBLIC ${PANDA_BINARY_ROOT}) + panda_add_sanitizers(TARGET aspt_converter SANITIZERS ${PANDA_SANITIZERS_LIST}) + +panda_gen_options(TARGET aspt_converter YAML_FILE options.yaml GENERATED_HEADER + aspt_converter_options.h) diff --git a/tools/sampler/README.md b/tools/sampler/README.md index 201f300ec..450358270 100644 --- a/tools/sampler/README.md +++ b/tools/sampler/README.md @@ -17,12 +17,21 @@ ${BUILD_DIR}/bin/es2panda ${BUILD_SOURCE}/plugins/ets/tests/runtime/tooling.samp ${BUILD_DIR}/bin/ark --load-runtimes=ets --boot-panda-files=${BUILD_DIR}/plugins/ets/etsstdlib.abc --sampling-profiler-enable --sampling-profiler-interval=200 --sampling-profiler-output-file=${BUILD_DIR}/outfile.aspt ${BUILD_DIR}/sampling_app.abc ETSGLOBAL::main # convert sample dump to csv -${BUILD_DIR}/bin/aspt_converter ${BUILD_DIR}/outfile.aspt ${BUILD_DIR}/traceout.csv +${BUILD_DIR}/bin/aspt_converter --input=${BUILD_DIR}/outfile.aspt --output=${BUILD_DIR}/traceout.csv # generate flamegrath svg ${BUILD_DIR}/FlameGraph/flamegraph.pl ${BUILD_DIR}/traceout.csv > ${BUILD_DIR}/out.svg ``` +In current implementation of aspt_converter two vizual modes are supported: +```bash +# convert sample dump to single csv with separating samples by threads +${BUILD_DIR}/bin/aspt_converter --input=${BUILD_DIR}/outfile.aspt --output=${BUILD_DIR}/traceout.csv --thread-separation-by-tid + +# convert sample dump many csv files with one thread per file. Name of ouput file will concatinate with thread_id +${BUILD_DIR}/bin/aspt_converter --input=${BUILD_DIR}/outfile.aspt --output=${BUILD_DIR}/traceout.csv --thread-separation-by-csv +``` + ## Limitations 1. Samples can be lost but not necessarily in case if signal was sent to interpreter thread while it was in the bridge and frame was not constructed yet. (Relevant to profiling with JIT or ArkTS NAPI) 1.1 You can track lost samples by adding options to runtime `--log-level=debug --log-components=profiler` diff --git a/tools/sampler/args_parser.h b/tools/sampler/args_parser.h new file mode 100644 index 000000000..bc9734bdc --- /dev/null +++ b/tools/sampler/args_parser.h @@ -0,0 +1,98 @@ +/* + * Copyright (c) 2021-2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_TOOLS_SAMPLER_ARGS_PARSER_H +#define PANDA_TOOLS_SAMPLER_ARGS_PARSER_H + +#include "tools/sampler/panda_gen_options/generated/aspt_converter_options.h" + +namespace panda::tooling::sampler { + +class ArgsParser { +public: + bool Parse(panda::Span args) + { + options_.AddOptions(&parser_); + + if (!parser_.Parse(args.Size(), args.Data())) { + std::cerr << parser_.GetErrorString(); + return false; + } + + auto err = options_.Validate(); + if (err) { + std::cerr << err.value().GetMessage() << std::endl; + return false; + } + + if (options_.GetInput().empty()) { + std::cerr << "[Error:] Input file is not set" << std::endl; + return false; + } + + if (options_.GetOutput().empty()) { + std::cerr << "[Error:] Output file is not set" << std::endl; + return false; + } + + if (!panda::os::IsFileExists(options_.GetInput())) { + std::cerr << "[Error:] File \"" << options_.GetInput() << "\" not found." << std::endl; + return false; + } + + if (options_.IsThreadSeparationByCsv() && options_.IsThreadSeparationByTid()) { + std::cerr << "[Error:] Mutually exclusive conditions have been chosen." << std::endl; + return false; + } + + return true; + } + + const Options &GetOptions() const + { + return options_; + } + + DumpType GetDumpType() const + { + auto dump_type = DumpType::WITHOUT_THREAD_SEPARATION; + if (options_.IsThreadSeparationByCsv()) { + dump_type = DumpType::THREAD_SEPARATION_BY_CSV; + } else if (options_.IsThreadSeparationByTid()) { + dump_type = DumpType::THREAD_SEPARATION_BY_TID; + } + + return dump_type; + } + + void Help() const + { + std::cerr << "AsptConverter usage: " << std::endl; + std::cerr << "${BUILD_DIR}/bin/aspt_converter " + << "--input= " + << "--output= " + << "[OPTIONS]" << std::endl; + std::cerr << "optional arguments:" << std::endl; + std::cerr << parser_.GetHelpString() << std::endl; + } + +private: + PandArgParser parser_; + Options options_ {""}; +}; + +} // namespace panda::tooling::sampler + +#endif // PANDA_TOOLS_SAMPLER_ARGS_PARSER_H diff --git a/tools/sampler/aspt_convert.cpp b/tools/sampler/aspt_convert.cpp index 7beebe4b9..012bf5944 100644 --- a/tools/sampler/aspt_convert.cpp +++ b/tools/sampler/aspt_convert.cpp @@ -15,29 +15,29 @@ #include "libpandabase/utils/logger.h" #include "tools/sampler/aspt_converter.h" +#include "tools/sampler/args_parser.h" namespace panda::tooling::sampler { int Main(int argc, const char **argv) { panda::Span sp(argv, argc); - if (sp.Size() != 3) { - std::cerr << "Error: Wrong amount of arguments." << std::endl - << "Usage: " << sp[0] << " input.aspt output" << std::endl; - return 1; - } - - const char *filename = sp[1]; - const char *out_filename = sp[2]; - if (!panda::os::IsFileExists(filename)) { - std::cerr << "Error: file \"" << filename << "\" not found." << std::endl; + ArgsParser parser; + if (!parser.Parse(sp)) { + parser.Help(); return 1; } + const Options &cli_options = parser.GetOptions(); Logger::InitializeStdLogging(Logger::Level::INFO, Logger::ComponentMaskFromString("profiler")); - AsptConverter conv(filename); + std::string input_filename = cli_options.GetInput(); + std::string output_filename = cli_options.GetOutput(); + + DumpType dump_type = parser.GetDumpType(); + + AsptConverter conv(input_filename.c_str(), dump_type); if (conv.CollectTracesStats() == 0) { LOG(ERROR, PROFILER) << "No samples found in file"; return 1; @@ -46,7 +46,7 @@ int Main(int argc, const char **argv) LOG(ERROR, PROFILER) << "No modules found in file, names would not be resolved"; } conv.BuildMethodsMap(); - conv.DumpResolvedTracesAsCSV(out_filename); + conv.DumpResolvedTracesAsCSV(output_filename.c_str()); return 0; } diff --git a/tools/sampler/aspt_converter.h b/tools/sampler/aspt_converter.h index 0843d1bdc..edd750a51 100644 --- a/tools/sampler/aspt_converter.h +++ b/tools/sampler/aspt_converter.h @@ -16,31 +16,24 @@ #ifndef PANDA_TOOLS_SAMPLER_CONVERTER_IMPL_H #define PANDA_TOOLS_SAMPLER_CONVERTER_IMPL_H -#include - #include "libpandabase/os/filesystem.h" #include "libpandabase/utils/utf.h" -#include "libpandafile/class_data_accessor-inl.h" -#include "libpandafile/file-inl.h" #include "runtime/tooling/sampler/sample_info.h" #include "runtime/tooling/sampler/sample_reader-inl.h" +#include "tools/sampler/trace_dumper.h" namespace panda::tooling::sampler { class AsptConverter { public: using StackTraceMap = std::unordered_map; - using ModuleMap = std::unordered_map>; - using MethodMap = std::unordered_map>; - - explicit AsptConverter(const char *filename) : reader_(filename) {} - ~AsptConverter() = default; - const StackTraceMap &GetStackTraces() const + explicit AsptConverter(const char *filename, DumpType dump_type = DumpType::WITHOUT_THREAD_SEPARATION) + : dump_type_(dump_type), reader_(filename) { - return stack_traces_; } + ~AsptConverter() = default; size_t CollectTracesStats() { @@ -50,6 +43,12 @@ public: SampleInfo sample; while (reader_.GetNextSample(&sample)) { ++sample_counter; + + if (dump_type_ == DumpType::WITHOUT_THREAD_SEPARATION) { + // NOTE: zeroing thread_id to make samples indistinguishable in mode without thread separation + sample.thread_info.thread_id = 0; + } + auto it = stack_traces_.find(sample); if (it == stack_traces_.end()) { stack_traces_.insert({sample, 1}); @@ -57,7 +56,6 @@ public: } ++it->second; } - return sample_counter; } @@ -80,36 +78,26 @@ public: bool DumpResolvedTracesAsCSV(const char *filename) { - std::ofstream stream(filename); + std::unique_ptr dumper; + + switch (dump_type_) { + case DumpType::WITHOUT_THREAD_SEPARATION: + case DumpType::THREAD_SEPARATION_BY_TID: + dumper = std::make_unique(filename, dump_type_, &modules_map_, &methods_map_); + break; + case DumpType::THREAD_SEPARATION_BY_CSV: + dumper = std::make_unique(filename, &modules_map_, &methods_map_); + break; + default: + UNREACHABLE(); + } for (auto &[sample, count] : stack_traces_) { - ASSERT(sample.managed_stack_size <= SampleInfo::MAX_STACK_DEPTH); - for (size_t i = sample.managed_stack_size; i-- > 0;) { - uintptr_t pf_id = sample.managed_stack[i].panda_file_ptr; - uint64_t file_id = sample.managed_stack[i].file_id; - const panda_file::File *pf = - modules_map_.find(pf_id) != modules_map_.end() ? modules_map_[pf_id].get() : nullptr; - const std::string full_method_name = ResolveName(pf, file_id); - stream << full_method_name << "; "; - } - stream << count << "\n"; + ASSERT(sample.stack_info.managed_stack_size <= SampleInfo::StackInfo::MAX_STACK_DEPTH); + dumper->DumpTraces(sample, count); } - return true; } - std::string ResolveName(const panda_file::File *pf, uint64_t file_id) const - { - if (pf == nullptr) { - return std::string("__unknown_module::" + std::to_string(file_id)); - } - - auto it = methods_map_.find(pf); - if (it != methods_map_.end()) { - return it->second.at(file_id); - } - return pf->GetFilename() + "::__unknown_" + std::to_string(file_id); - } - void BuildMethodsMap() { for (const auto &pf_pair : modules_map_) { @@ -138,6 +126,7 @@ public: NO_MOVE_SEMANTIC(AsptConverter); private: + DumpType dump_type_; SampleReader reader_; ModuleMap modules_map_; MethodMap methods_map_; diff --git a/tools/sampler/options.yaml b/tools/sampler/options.yaml new file mode 100644 index 000000000..5a147a65f --- /dev/null +++ b/tools/sampler/options.yaml @@ -0,0 +1,38 @@ +# Copyright (c) 2021-2022 Huawei Device Co., Ltd. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +module: + name: aspt_converter + namespace: panda::tooling::sampler + +options: +- name: thread-separation-by-csv + type: bool + default: false + description: Separates threads by creating csv files for each thread + +- name: thread-separation-by-tid + type: bool + default: false + description: Separates threads by thread_id in a single csv file + +- name: input + type: std::string + default: "" + description: Path to input aspt file + +- name: output + type: std::string + default: "" + description: Path to ouput csv file + \ No newline at end of file diff --git a/tools/sampler/trace_dumper.h b/tools/sampler/trace_dumper.h new file mode 100644 index 000000000..04742de95 --- /dev/null +++ b/tools/sampler/trace_dumper.h @@ -0,0 +1,165 @@ +/* + * Copyright (c) 2021-2022 Huawei Device Co., Ltd. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef PANDA_TOOLS_SAMPLER_TRACE_DUMPER_H +#define PANDA_TOOLS_SAMPLER_TRACE_DUMPER_H + +#include +#include + +#include "libpandabase/macros.h" +#include "libpandabase/utils/logger.h" +#include "libpandafile/file-inl.h" +#include "libpandafile/class_data_accessor-inl.h" + +#include "runtime/tooling/sampler/sample_info.h" + +namespace panda::tooling::sampler { + +using ModuleMap = std::unordered_map>; +using MethodMap = std::unordered_map>; + +enum class DumpType { + WITHOUT_THREAD_SEPARATION, // Creates single csv file without separating threads + THREAD_SEPARATION_BY_TID, // Creates single csv file with separating threads by thread_id + THREAD_SEPARATION_BY_CSV // Creates csv file for each thread +}; + +class TraceDumper { +public: + explicit TraceDumper(ModuleMap *modules_map, MethodMap *methods_map) + : modules_map_(modules_map), methods_map_(methods_map) + { + } + virtual ~TraceDumper() = default; + + virtual std::ofstream &ResolveStream(const SampleInfo &sample) = 0; + + void DumpTraces(const SampleInfo &sample, size_t count) + { + std::ofstream &stream = ResolveStream(sample); + + for (size_t i = sample.stack_info.managed_stack_size; i-- > 0;) { + uintptr_t pf_id = sample.stack_info.managed_stack[i].panda_file_ptr; + uint64_t file_id = sample.stack_info.managed_stack[i].file_id; + + const panda_file::File *pf = nullptr; + auto it = modules_map_->find(pf_id); + if (it != modules_map_->end()) { + pf = it->second.get(); + } + + const std::string full_method_name = ResolveName(pf, file_id); + + stream << full_method_name << "; "; + } + stream << count << "\n"; + } + + std::string ResolveName(const panda_file::File *pf, uint64_t file_id) const + { + if (pf == nullptr) { + return std::string("__unknown_module::" + std::to_string(file_id)); + } + + auto it = methods_map_->find(pf); + if (it != methods_map_->end()) { + return it->second.at(file_id); + } + return pf->GetFilename() + "::__unknown_" + std::to_string(file_id); + } + + NO_COPY_SEMANTIC(TraceDumper); + NO_MOVE_SEMANTIC(TraceDumper); + +private: + ModuleMap *modules_map_ {nullptr}; + MethodMap *methods_map_ {nullptr}; +}; + +class SingleCSVDumper final : public TraceDumper { +public: + explicit SingleCSVDumper(const char *filename, DumpType option, ModuleMap *modules_map, MethodMap *methods_map) + : TraceDumper(modules_map, methods_map) + { + stream_ = std::ofstream(filename); + option_ = option; + } + ~SingleCSVDumper() override = default; + + std::ofstream &ResolveStream(const SampleInfo &sample) override + { + if (option_ == DumpType::THREAD_SEPARATION_BY_TID) { + stream_ << "thread_id = " << sample.thread_info.thread_id << "; "; + } + return stream_; + } + + NO_COPY_SEMANTIC(SingleCSVDumper); + NO_MOVE_SEMANTIC(SingleCSVDumper); + +private: + std::ofstream stream_; + DumpType option_; +}; + +class MultipleCSVDumper final : public TraceDumper { +public: + explicit MultipleCSVDumper(const char *filename, ModuleMap *modules_map, MethodMap *methods_map) + : TraceDumper(modules_map, methods_map), filename_(filename) + { + } + ~MultipleCSVDumper() override = default; + + static std::string AddThreadIdToFilename(const std::string &filename, const uint32_t thread_id) + { + std::string filename_with_thread_id(filename); + std::size_t pos = filename_with_thread_id.find("csv"); + if (pos == std::string::npos) { + LOG(FATAL, PROFILER) << "Incorrect output filename, *.csv format expected"; + } + filename_with_thread_id.insert(pos - 1, std::to_string(thread_id)); + return filename_with_thread_id; + } + + std::ofstream &ResolveStream(const SampleInfo &sample) override + { + auto it = thread_id_map_.find(sample.thread_info.thread_id); + if (it == thread_id_map_.end()) { + std::string filename_with_thread_id = AddThreadIdToFilename(filename_, sample.thread_info.thread_id); + + auto return_pair = + thread_id_map_.insert({sample.thread_info.thread_id, std::ofstream(filename_with_thread_id)}); + it = return_pair.first; + + auto is_success_insert = return_pair.second; + if (!is_success_insert) { + LOG(FATAL, PROFILER) << "Failed while insert in unordored_map"; + } + } + return it->second; + } + + NO_COPY_SEMANTIC(MultipleCSVDumper); + NO_MOVE_SEMANTIC(MultipleCSVDumper); + +private: + std::string filename_; + std::unordered_map thread_id_map_; +}; + +} // namespace panda::tooling::sampler + +#endif // PANDA_TOOLS_SAMPLER_TRACE_DUMPER_H -- Gitee