From 8113ab80ec16789139be89353efb5e618ca91e43 Mon Sep 17 00:00:00 2001 From: Lixy Date: Thu, 20 Oct 2022 17:32:08 +0800 Subject: [PATCH] openpose_CPU modify --- official/cv/openpose/README.md | 43 ++- official/cv/openpose/default_config.yaml | 2 +- official/cv/openpose/quick_start.py | 391 +++++++++++++++++++++++ 3 files changed, 411 insertions(+), 25 deletions(-) create mode 100644 official/cv/openpose/quick_start.py diff --git a/official/cv/openpose/README.md b/official/cv/openpose/README.md index 08e5b835c..041bbe872 100644 --- a/official/cv/openpose/README.md +++ b/official/cv/openpose/README.md @@ -74,8 +74,8 @@ For FP16 operators, if the input data type is FP32, the backend of MindSpore wil # [Environment Requirements](#contents) -- Hardware (Ascend) - - Prepare hardware environment with Ascend. +- Hardware (CPU) + - Prepare hardware environment with CPU. - Framework - [MindSpore](https://www.mindspore.cn/install/en) - Download the VGG19 model of the MindSpore version: @@ -92,15 +92,12 @@ After installing MindSpore via the official website, you can start training and # run training example python train.py --imgpath_train /home/DataSet/coco/train2017 --jsonpath_train /home/DataSet/coco/annotations/person_keypoints_train2017.json --maskpath_train /home/DataSet/coco/ignore_mask_train --vgg_path /home/model/openpose/vgg19-0-97_5004.ckpt > train.log 2>&1 & - # run distributed training example - bash run_distribute_train.sh [RANK_TABLE_FILE] [IMGPATH_TRAIN] [JSONPATH_TRAIN] [MASKPATH_TRAIN] [VGG_PATH] - # example: bash run_distribute_train.sh ~/hccl_8p.json /home/DataSet/coco/train2017 /home/DataSet/coco/annotations/person_keypoints_train2017.json /home/DataSet/coco/ignore_mask_train /home/model/openpose/vgg19-0-97_5004.ckpt + # run evaluation example + python eval.py --model_path /home/model/openpose/ckpt/0-60_5236.ckpt --imgpath_val /home/DataSet/coco/val2017 --ann /home/DataSet/coco/annotations/person_keypoints_val2017.json > eval.log 2>&1 & + + # run quick_start example + python quick_start.py --model_path /home/model/openpose/ckpt/0-60_5236.ckpt - # run evaluation example - python eval.py --model_path /home/model/openpose/ckpt/0-8_663.ckpt --imgpath_val /home/DataSet/coco/val2017 --ann /home/DataSet/coco/annotations/person_keypoints_val2017.json > eval.log 2>&1 & - OR - bash scripts/run_eval_ascend.sh [MODEL_PATH] [IMGPATH_VAL] [ANN] - # example: bash scripts/run_eval_ascend.sh /home/model/openpose/ckpt/0-8_663.ckpt /home/DataSet/coco/val2017 /home/DataSet/coco/annotations/person_keypoints_val2017.json ``` [RANK_TABLE_FILE] is the path of the multi-card information configuration table in the environment. The configuration table can be automatically generated by the tool [hccl_tool](https://gitee.com/mindspore/models/tree/master/utils/hccl_tools). @@ -132,6 +129,7 @@ After installing MindSpore via the official website, you can start training and ├── eval.py // evaluation script ├── mindspore_hub_config.py // hub config file ├── default_config.yaml // config file + ├── quick_start.py // quick start file ``` ## [Script Parameters](#contents) @@ -164,7 +162,7 @@ For more configuration details, please refer the script `default_config.yaml`. ### Training -- running on Ascend +- running on CPU ```python python train.py --imgpath_train /home/DataSet/coco/train2017 --jsonpath_train /home/DataSet/coco/annotations/person_keypoints_train2017.json --maskpath_train /home/DataSet/coco/ignore_mask_train --vgg_path /home/model/openpose/vgg19-0-97_5004.ckpt > train.log 2>&1 & @@ -245,16 +243,13 @@ For more configuration details, please refer the script `default_config.yaml`. ### Evaluation -- running on Ascend +- running on CPU Before running the command below, please check the checkpoint path used for evaluation. Please set the checkpoint path to be the absolute full path, e.g., "username/openpose/outputs/\*time*\/0-6_30000.ckpt". ```python # run evaluation example python eval.py --model_path /home/model/openpose/ckpt/0-8_663.ckpt --imgpath_val /home/DataSet/coco/val2017 --ann /home/DataSet/coco/annotations/person_keypoints_val2017.json > eval.log 2>&1 & - OR - bash scripts/run_eval_ascend.sh [MODEL_PATH] [IMGPATH_VAL] [ANN] - # example: bash scripts/run_eval_ascend.sh /home/model/openpose/ckpt/0-8_663.ckpt /home/DataSet/coco/val2017 /home/DataSet/coco/annotations/person_keypoints_val2017.json ``` The above python command will run in the background. You can view the results through the file "eval.log". The accuracy of the test dataset will be as follows: @@ -293,16 +288,16 @@ Data storage method is the same as training ### Evaluation Performance -| Parameters | Ascend +| Parameters | CPU | -------------------------- | ----------------------------------------------------------- | Model Version | openpose -| Resource | Ascend 910; CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8 -| uploaded Date | 12/14/2020 (month/day/year) -| MindSpore Version | 1.0.1 -| Training Parameters | epoch=60(1pcs)/80(8pcs), steps=30k(1pcs)/5k(8pcs), batch_size=10, init_lr=0.0001 -| Optimizer | Adam(1pcs)/Momentum(8pcs) +| Resource | CPU 2.60GHz, 192cores; Memory 755G; OS Euler2.8 +| uploaded Date | 10/14/2022 (month/day/year) +| MindSpore Version | 1.8.1 +| Training Parameters | epoch=60(1pcs), steps=30k(1pcs), batch_size=10, init_lr=0.0001 +| Optimizer | Adam(1pcs) | Loss Function | MSE | outputs | pose -| Speed | 1pcs: 35fps, 8pcs: 230fps -| Total time | 1pcs: 22.5h, 8pcs: 5.1h -| Checkpoint for Fine tuning | 602.33M (.ckpt file) +| Speed | 1pcs: 35fps, +| Total time | 1pcs: 22.5h +| Checkpoint for Fine tuning | 801.85M (.ckpt file) diff --git a/official/cv/openpose/default_config.yaml b/official/cv/openpose/default_config.yaml index 95685a655..8ef6f5983 100644 --- a/official/cv/openpose/default_config.yaml +++ b/official/cv/openpose/default_config.yaml @@ -8,7 +8,7 @@ checkpoint_url: "" data_path: "/cache/data" output_path: "/cache/train" load_path: "/cache/checkpoint_path" -device_target: "Ascend" +device_target: "CPU" enable_profiling: False # ====================================================================================== diff --git a/official/cv/openpose/quick_start.py b/official/cv/openpose/quick_start.py new file mode 100644 index 000000000..c1a966581 --- /dev/null +++ b/official/cv/openpose/quick_start.py @@ -0,0 +1,391 @@ +# Copyright 2020 Huawei Technologies Co., Ltd + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# ============================================================================ + +import os +import warnings +import numpy as np +import cv2 +from scipy.ndimage import gaussian_filter +from mindspore import context, Tensor +from mindspore.train.serialization import load_checkpoint, load_param_into_net +from mindspore.communication.management import init +from mindspore.common import dtype as mstype +from src.openposenet import OpenPoseNet +from src.model_utils.config import config, JointType +from src.model_utils.moxing_adapter import moxing_wrapper +from src.model_utils.device_adapter import get_device_id, get_rank_id, get_device_num + + +warnings.filterwarnings("ignore") +devid = get_device_id() +context.set_context(mode=context.GRAPH_MODE, + device_target=config.device_target, save_graphs=False, device_id=devid) +show_gt = 0 + + +def load_model(test_net, model_path): + assert os.path.exists(model_path) + param_dict = load_checkpoint(model_path) + param_dict_new = {} + for key, values in param_dict.items(): + + if key.startswith('moment'): + continue + elif key.startswith('network'): + param_dict_new[key[8:]] = values + + load_param_into_net(test_net, param_dict_new) + + +def preprocess(img): + x_data = img.astype('f') + x_data /= 255 + x_data -= 0.5 + x_data = x_data.transpose(2, 0, 1)[None] + return x_data + + +def getImgsPath(img_dir_path): + filepaths = [] + dirpaths = [] + pathName = img_dir_path + + for root, dirs, files in os.walk(pathName): + for file in files: + file_path = os.path.join(root, file) + filepaths.append(file_path) + for d in dirs: + dir_path = os.path.join(root, d) + dirpaths.append(dir_path) + return filepaths + + +def compute_optimal_size(orig_img, img_size, stride=8): + orig_img_h, orig_img_w, _ = orig_img.shape + aspect = orig_img_h / orig_img_w + if orig_img_h < orig_img_w: + img_h = img_size + img_w = np.round(img_size / aspect).astype(int) + surplus = img_w % stride + if surplus != 0: + img_w += stride - surplus + else: + img_w = img_size + img_h = np.round(img_size * aspect).astype(int) + surplus = img_h % stride + if surplus != 0: + img_h += stride - surplus + return (img_w, img_h) + + +def compute_peaks_from_heatmaps(heatmaps): + + heatmaps = heatmaps[:-1] + + all_peaks = [] + peak_counter = 0 + for i, heatmap in enumerate(heatmaps): + heatmap = gaussian_filter(heatmap, sigma=config.gaussian_sigma) + + map_left = np.zeros(heatmap.shape) + map_right = np.zeros(heatmap.shape) + map_top = np.zeros(heatmap.shape) + map_bottom = np.zeros(heatmap.shape) + + map_left[1:, :] = heatmap[:-1, :] + map_right[:-1, :] = heatmap[1:, :] + map_top[:, 1:] = heatmap[:, :-1] + map_bottom[:, :-1] = heatmap[:, 1:] + + peaks_binary = np.logical_and.reduce(( + heatmap > config.heatmap_peak_thresh, + heatmap > map_left, + heatmap > map_right, + heatmap > map_top, + heatmap > map_bottom, + )) + + peaks = zip(np.nonzero(peaks_binary)[1], np.nonzero(peaks_binary)[0]) + + peaks_with_score = [(i,) + peak_pos + (heatmap[peak_pos[1], peak_pos[0]],) for peak_pos in peaks] + + peaks_id = range(peak_counter, peak_counter + len(peaks_with_score)) + peaks_with_score_and_id = [peaks_with_score[i] + (peaks_id[i],) for i in range(len(peaks_id))] + + peak_counter += len(peaks_with_score_and_id) + all_peaks.append(peaks_with_score_and_id) + all_peaks = np.array([peak for peaks_each_category in all_peaks for peak in peaks_each_category]) + + return all_peaks + + +def compute_candidate_connections(paf, cand_a, cand_b, img_len, params_): + candidate_connections = [] + for joint_a in cand_a: + for joint_b in cand_b: + vector = joint_b[:2] - joint_a[:2] + norm = np.linalg.norm(vector) + if norm == 0: + continue + ys = np.linspace(joint_a[1], joint_b[1], num=params_.n_integ_points) + xs = np.linspace(joint_a[0], joint_b[0], num=params_.n_integ_points) + integ_points = np.stack([ys, xs]).T.round().astype('i') + + paf_in_edge = np.hstack([paf[0][np.hsplit(integ_points, 2)], paf[1][np.hsplit(integ_points, 2)]]) + unit_vector = vector / norm + inner_products = np.dot(paf_in_edge, unit_vector) + integ_value = inner_products.sum() / len(inner_products) + integ_value_with_dist_prior = integ_value + min(params_.limb_length_ratio * img_len / norm - + params_.length_penalty_value, 0) + n_valid_points = sum(inner_products > params_.inner_product_thresh) + if n_valid_points > params_.n_integ_points_thresh and integ_value_with_dist_prior > 0: + candidate_connections.append([int(joint_a[3]), int(joint_b[3]), integ_value_with_dist_prior]) + candidate_connections = sorted(candidate_connections, key=lambda x: x[2], reverse=True) + return candidate_connections + + +def compute_connections(pafs, all_peaks, img_len, params_): + all_connections = [] + for i in range(len(params_.limbs_point)): + paf_index = [i * 2, i * 2 + 1] + paf = pafs[paf_index] # shape: (2, 320, 320) + limb_point = params_.limbs_point[i] # example: [, ] + cand_a = all_peaks[all_peaks[:, 0] == limb_point[0]][:, 1:] + cand_b = all_peaks[all_peaks[:, 0] == limb_point[1]][:, 1:] + + if cand_a.shape[0] > 0 and cand_b.shape[0] > 0: + candidate_connections = compute_candidate_connections(paf, cand_a, cand_b, img_len, params_) + + connections = np.zeros((0, 3)) + + for index_a, index_b, score in candidate_connections: + if index_a not in connections[:, 0] and index_b not in connections[:, 1]: + connections = np.vstack([connections, [index_a, index_b, score]]) + if len(connections) >= min(len(cand_a), len(cand_b)): + break + all_connections.append(connections) + else: + all_connections.append(np.zeros((0, 3))) + return all_connections + +def grouping_key_points(all_connections, candidate_peaks, params_): + subsets = -1 * np.ones((0, 20)) + + for l, connections in enumerate(all_connections): + joint_a, joint_b = params_.limbs_point[l] + for ind_a, ind_b, score in connections[:, :3]: + ind_a, ind_b = int(ind_a), int(ind_b) + joint_found_cnt = 0 + joint_found_subset_index = [-1, -1] + for subset_ind, subset in enumerate(subsets): + + if subset[joint_a] == ind_a or subset[joint_b] == ind_b: + joint_found_subset_index[joint_found_cnt] = subset_ind + joint_found_cnt += 1 + + if joint_found_cnt == 1: + + found_subset = subsets[joint_found_subset_index[0]] + if found_subset[joint_b] != ind_b: + found_subset[joint_b] = ind_b + found_subset[-1] += 1 # increment joint count + found_subset[-2] += candidate_peaks[ind_b, 3] + score + + + elif joint_found_cnt == 2: + + found_subset_1 = subsets[joint_found_subset_index[0]] + found_subset_2 = subsets[joint_found_subset_index[1]] + + membership = ((found_subset_1 >= 0).astype(int) + (found_subset_2 >= 0).astype(int))[:-2] + if not np.any(membership == 2): # merge two subsets when no duplication + found_subset_1[:-2] += found_subset_2[:-2] + 1 # default is -1 + found_subset_1[-2:] += found_subset_2[-2:] + found_subset_1[-2] += score + subsets = np.delete(subsets, joint_found_subset_index[1], axis=0) + else: + if found_subset_1[joint_a] == -1: + found_subset_1[joint_a] = ind_a + found_subset_1[-1] += 1 + found_subset_1[-2] += candidate_peaks[ind_a, 3] + score + elif found_subset_1[joint_b] == -1: + found_subset_1[joint_b] = ind_b + found_subset_1[-1] += 1 + found_subset_1[-2] += candidate_peaks[ind_b, 3] + score + if found_subset_2[joint_a] == -1: + found_subset_2[joint_a] = ind_a + found_subset_2[-1] += 1 + found_subset_2[-2] += candidate_peaks[ind_a, 3] + score + elif found_subset_2[joint_b] == -1: + found_subset_2[joint_b] = ind_b + found_subset_2[-1] += 1 + found_subset_2[-2] += candidate_peaks[ind_b, 3] + score + + elif joint_found_cnt == 0 and l != 9 and l != 13: + row = -1 * np.ones(20) + row[joint_a] = ind_a + row[joint_b] = ind_b + row[-1] = 2 + row[-2] = sum(candidate_peaks[[ind_a, ind_b], 3]) + score + subsets = np.vstack([subsets, row]) + elif joint_found_cnt >= 3: + pass + + # delete low score subsets + keep = np.logical_and(subsets[:, -1] >= params_.n_subset_limbs_thresh, + subsets[:, -2] / subsets[:, -1] >= params_.subset_score_thresh) + subsets = subsets[keep] + return subsets + + +def subsets_to_pose_array(subsets, all_peaks): + person_pose_array = [] + for subset in subsets: + joints = [] + for joint_index in subset[:18].astype('i'): + if joint_index >= 0: + joint = all_peaks[joint_index][1:3].tolist() + joint.append(2) + joints.append(joint) + else: + joints.append([0, 0, 0]) + person_pose_array.append(np.array(joints)) + person_pose_array = np.array(person_pose_array) + return person_pose_array + +def detect(img, network): + orig_img = img.copy() + orig_img_h, orig_img_w, _ = orig_img.shape + + input_w, input_h = compute_optimal_size(orig_img, config.inference_img_size) # 368 + map_w, map_h = compute_optimal_size(orig_img, config.inference_img_size) + + resized_image = cv2.resize(orig_img, (input_w, input_h)) + x_data = preprocess(resized_image) + x_data = Tensor(x_data, mstype.float32) + x_data.requires_grad = False + + logit_pafs, logit_heatmap = network(x_data) + + logit_pafs = logit_pafs[-1].asnumpy()[0] + logit_heatmap = logit_heatmap[-1].asnumpy()[0] + + pafs = np.zeros((logit_pafs.shape[0], map_h, map_w)) + for i in range(logit_pafs.shape[0]): + pafs[i] = cv2.resize(logit_pafs[i], (map_w, map_h)) + if show_gt: + save_path = "./test_output/" + str(i) + "pafs.png" + cv2.imwrite(save_path, pafs[i]*255) + + heatmaps = np.zeros((logit_heatmap.shape[0], map_h, map_w)) + for i in range(logit_heatmap.shape[0]): + heatmaps[i] = cv2.resize(logit_heatmap[i], (map_w, map_h)) + if show_gt: + save_path = "./test_output/" + str(i) + "heatmap.png" + cv2.imwrite(save_path, heatmaps[i]*255) + + all_peaks = compute_peaks_from_heatmaps(heatmaps) + if all_peaks.shape[0] == 0: + return np.empty((0, len(JointType), 3)), np.empty(0) + all_connections = compute_connections(pafs, all_peaks, map_w, config) + subsets = grouping_key_points(all_connections, all_peaks, config) + all_peaks[:, 1] *= orig_img_w / map_w + all_peaks[:, 2] *= orig_img_h / map_h + poses = subsets_to_pose_array(subsets, all_peaks) + scores = subsets[:, -2] + + return poses, scores + +def draw_person_pose(orig_img, poses): + orig_img = cv2.cvtColor(orig_img, cv2.COLOR_BGR2RGB) + if poses.shape[0] == 0: + return orig_img + + limb_colors = [ + [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255], [0, 170, 255], + [0, 85, 255], [255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0.], + [255, 0, 85], [170, 255, 0], [85, 255, 0], [170, 0, 255.], [0, 0, 255], + [0, 0, 255], [255, 0, 255], [170, 0, 255], [255, 0, 170], + ] + + joint_colors = [ + [255, 0, 0], [255, 85, 0], [255, 170, 0], [255, 255, 0], [170, 255, 0], + [85, 255, 0], [0, 255, 0], [0, 255, 85], [0, 255, 170], [0, 255, 255], + [0, 170, 255], [0, 85, 255], [0, 0, 255], [85, 0, 255], [170, 0, 255], + [255, 0, 255], [255, 0, 170], [255, 0, 85]] + + canvas = orig_img.copy() + + # limbs + for pose in poses.round().astype('i'): + for i, (limb, color) in enumerate(zip(config.limbs_point, limb_colors)): + if i not in (9, 13): # don't show ear-shoulder connection + limb_ind = np.array(limb) + if np.all(pose[limb_ind][:, 2] != 0): + joint1, joint2 = pose[limb_ind][:, :2] + cv2.line(canvas, tuple(joint1), tuple(joint2), color, 2) + + # joints + for pose in poses.round().astype('i'): + for i, ((x, y, v), color) in enumerate(zip(pose, joint_colors)): + if v != 0: + cv2.circle(canvas, (x, y), 3, color, -1) + return canvas + + +def depreprocess(img): + x_data = img[0] + x_data += 0.5 + x_data *= 255 + x_data = x_data.astype('uint8') + x_data = x_data.transpose(1, 2, 0) + return x_data + +@moxing_wrapper(pre_process=None) +def go(): + config.rank = get_rank_id() + config.group_size = get_device_num() + + if config.is_distributed: + init() + config.rank = get_rank_id() + config.group_size = get_device_num() + if not os.path.exists(config.output_img_path): + os.mkdir(config.output_img_path) + network = OpenPoseNet(vgg_with_bn=config.vgg_with_bn) + network.set_train(False) + load_model(network, config.model_path) + + print("load models right") + + + img = cv2.imread("/home/DataSet/coco/val2017/000000000872.jpg") + + poses, scores = detect(img, network) + + + img = draw_person_pose(cv2.cvtColor(img, cv2.COLOR_BGR2RGB), poses) + + save_path = os.path.join(config.output_img_path, "quick_start"+".png") + cv2.imwrite(save_path, img) + + + + + + +if __name__ == "__main__": + + go() \ No newline at end of file -- Gitee