From 8b4931d071f3484d77842f4265cc2b000f9916ff Mon Sep 17 00:00:00 2001 From: LazyBusyYang Date: Mon, 12 Jun 2023 13:01:40 +0800 Subject: [PATCH] [Feature] Estimate mview sperson dataset in filesystem (#123) 1. Add `HummanSMCDataCovnerter`, converting SMCs to dataset in filesystem. 2. Add a tool `run_mview_sperson_estimator`, running estimation on the files above. Also this tool addresses the requirements mentioned in the [issue-118](https://github.com/openxrlab/xrmocap/issues/118). 3. Add datautils for better logger. --- ...humman_smc_data_converter_wo_perception.py | 8 + docs/en/tools/run_mview_sperson_estimator.md | 80 +++++ setup.cfg | 2 +- tools/prepare_dataset.py | 19 +- tools/process_smc.py | 18 +- tools/run_mview_sperson_estimator.py | 232 ++++++++++++ xrmocap/data/data_converter/builder.py | 3 + .../humman_smc_data_converter.py | 333 ++++++++++++++++++ .../data_converter/panoptic_data_converter.py | 6 +- xrmocap/utils/date_utils.py | 35 ++ 10 files changed, 718 insertions(+), 18 deletions(-) create mode 100644 configs/modules/data/data_converter/humman_smc_data_converter_wo_perception.py create mode 100644 docs/en/tools/run_mview_sperson_estimator.md create mode 100644 tools/run_mview_sperson_estimator.py create mode 100644 xrmocap/data/data_converter/humman_smc_data_converter.py create mode 100644 xrmocap/utils/date_utils.py diff --git a/configs/modules/data/data_converter/humman_smc_data_converter_wo_perception.py b/configs/modules/data/data_converter/humman_smc_data_converter_wo_perception.py new file mode 100644 index 00000000..56446674 --- /dev/null +++ b/configs/modules/data/data_converter/humman_smc_data_converter_wo_perception.py @@ -0,0 +1,8 @@ +type = 'HummanSMCDataCovnerter' +data_root = 'xrmocap_data/humman_dataset' +bbox_detector = None +kps2d_estimator = None +batch_size = 1000 +view_idxs = [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] +meta_path = 'xrmocap_data/humman_dataset/xrmocap_meta' +visualize = False diff --git a/docs/en/tools/run_mview_sperson_estimator.md b/docs/en/tools/run_mview_sperson_estimator.md new file mode 100644 index 00000000..0348db1c --- /dev/null +++ b/docs/en/tools/run_mview_sperson_estimator.md @@ -0,0 +1,80 @@ +## Tool run_mview_sperson_estimator + + +- [Overview](#overview) +- [Argument: estimator_config](#argument-estimator_config) +- [Argument: output_dir](#argument-output_dir) +- [Argument: disable_log_file](#argument-disable_log_file) +- [Argument: visualize](#argument-visualize) +- [Example](#example) + +### Overview + +If you don't want to write code to call the `MultiViewSinglePersonSMPLEstimator` yourself, it's okay. We provide [this tool](../../../tools/run_mview_sperson_estimator.py) that can be run under a specific directory structure. Before using this tool, you need to organize the files required by the estimator according to the rules below. + +``` +your_dataset_root +└── xrmocap_meta + ├── dataset_name.txt + ├── scene_0 + │   ├── camera_parameters + │   │   ├── fisheye_param_00.json + │   │   ├── fisheye_param_01.json + │   │   └── ... + │   ├── image_list_view_00.txt + │   ├── image_list_view_01.txt + │   ├── ... + │   ├── images_view_00 + │   │   ├── 000000.jpg + │   │   ├── 000001.jpg + │   │   └── ... + │   ├── images_view_01 + │   │   └── ... + │   └── ... + └── scene_1 +    └── ... +``` +`fisheye_param_{view_idx}.json` is a json file for XRPrimer FisheyeCameraParameter, please refer to [XRPrimer docs](https://github.com/openxrlab/xrprimer/blob/main/docs/en/data_structure/camera.md) for details. +`image_list_view_{view_idx}.txt` is a list of image paths relative to your dataset root, here's an example. +``` +xrmocap_meta/scene_0/images_view_00/000000.jpg +xrmocap_meta/scene_0/images_view_00/000001.jpg +... +``` + +### Argument: estimator_config + +`estimator_config` is the path to a `MultiViewSinglePersonSMPLEstimator` config file. For more details, see docs for `MultiViewSinglePersonSMPLEstimator` and the docstring in [code](../../../xrmocap/estimation/mview_sperson_smpl_estimator.py). + +Also, you can find our prepared config files at `config/estimation/mview_sperson_smpl_estimator.py`. + +### Argument: data_root + +`data_root` is the path to the root directory of dataset. In the file tree example given above, `data_root` should point to `your_dataset_root`. + +### Argument: meta_path + +`meta_path` is the path to the meta data directory. In the file tree example given above, `meta_path` should point to `xrmocap_meta`. + +### Argument: output_dir + +`output_dir` is the path to the directory saving all possible output files, including multi-view keypoints2d, keypoints3d, SMPLData and visualization videos. + +### Argument: disable_log_file + +By default, disable_log_file is False and a log file named `{tool_name}_{time_str}.txt` will be written. Add `--disable_log_file` makes it True and the tool will only print log to console. + +### Argument: visualize + +By default, visualize is False. Add `--visualize` makes it True and the tool will visualize keypoints3d with an orbit camera, overlay projected keypoints3d on some views, and overlay SMPL meshes on one view. + +### Example + +Run the tool with visualization. + +```bash +python tools/mview_sperson_estimator.py \ + --data_root xrmocap_data/humman_dataset \ + --meta_path xrmocap_data/humman_dataset/xrmocap_meta \ + --visualize +``` diff --git a/setup.cfg b/setup.cfg index 5b902627..e2764f61 100644 --- a/setup.cfg +++ b/setup.cfg @@ -12,6 +12,6 @@ multi_line_output = 5 include_trailing_comma = true known_standard_library = pkg_resources,setuptools known_first_party = xrmocap -known_third_party =PIL,cv2,filterpy,matplotlib,mmcv,mmhuman3d,numpy,prettytable,pytest,pytorch3d,scipy,smplx,sphinx_rtd_theme,torch,torchvision,tqdm,xrprimer +known_third_party =PIL,cv2,dateutil,filterpy,matplotlib,mmcv,mmhuman3d,numpy,prettytable,pytest,pytorch3d,scipy,smplx,sphinx_rtd_theme,torch,torchvision,tqdm,xrprimer no_lines_before = STDLIB,LOCALFOLDER default_section = THIRDPARTY diff --git a/tools/prepare_dataset.py b/tools/prepare_dataset.py index 943ed38c..3d2698ea 100644 --- a/tools/prepare_dataset.py +++ b/tools/prepare_dataset.py @@ -1,6 +1,5 @@ # yapf: disable import argparse -import datetime import logging import mmcv import os @@ -9,6 +8,7 @@ from xrprimer.utils.path_utils import Existence, check_path_existence from xrmocap.data.data_converter.builder import build_data_converter +from xrmocap.utils.date_utils import get_datetime_local, get_str_from_datetime # yapf: enable @@ -17,15 +17,18 @@ def main(args): converter_config = dict(mmcv.Config.fromfile(args.converter_config)) if check_path_existence('logs', 'dir') == Existence.DirectoryNotExist: os.mkdir('logs') + filename = os.path.basename(__file__).split('.')[0] if not args.disable_log_file: - time_str = datetime.datetime.now().strftime('%Y-%m-%d_%H:%M:%S') - log_path = os.path.join('logs', f'converter_log_{time_str}.txt') + datetime = get_datetime_local() + time_str = get_str_from_datetime(datetime) + log_path = os.path.join('logs', f'{filename}_{time_str}.txt') logger = setup_logger( - logger_name=__name__, + logger_name=filename, logger_path=log_path, - logger_level=logging.DEBUG) + file_level=logging.DEBUG, + console_level=logging.INFO) else: - logger = setup_logger(logger_name=__name__) + logger = setup_logger(logger_name=filename) if len(args.data_root) > 0 and len(args.meta_path) > 0: logger.info('Taking paths from sys.argv.') converter_config['data_root'] = args.data_root @@ -41,8 +44,8 @@ def main(args): data_converter = build_data_converter(converter_config) data_converter.run(overwrite=args.overwrite) if not args.disable_log_file: - shutil.move( - log_path, + shutil.copy( + src=log_path, dst=os.path.join(converter_config['meta_path'], f'converter_log_{time_str}.txt')) diff --git a/tools/process_smc.py b/tools/process_smc.py index 7c50a0ff..dac692b9 100644 --- a/tools/process_smc.py +++ b/tools/process_smc.py @@ -1,6 +1,5 @@ # yapf: disable import argparse -import datetime import glob import mmcv import numpy as np @@ -11,7 +10,7 @@ visualize_smpl_calibration, ) from xrprimer.utils.ffmpeg_utils import array_to_images -from xrprimer.utils.log_utils import setup_logger +from xrprimer.utils.log_utils import logging, setup_logger from xrprimer.utils.path_utils import ( Existence, check_path_existence, prepare_output_path, ) @@ -21,11 +20,13 @@ from xrmocap.data_structure.smc_reader import SMCReader from xrmocap.io.camera import get_all_color_kinect_parameter_from_smc from xrmocap.transform.image.color import rgb2bgr +from xrmocap.utils.date_utils import get_datetime_local, get_str_from_datetime # yapf: enable def main(args): + filename = os.path.basename(__file__).split('.')[0] # check output path exist_result = check_path_existence(args.output_dir, 'dir') if exist_result == Existence.MissingParent: @@ -35,11 +36,16 @@ def main(args): file_name = args.smc_path.rsplit('/', 1)[-1] smc_name = file_name.rsplit('.', 1)[0] if not args.disable_log_file: - time_str = datetime.datetime.now().strftime('%Y.%m.%d_%H:%M:%S') - log_path = os.path.join(args.output_dir, f'{smc_name}_{time_str}.txt') - logger = setup_logger(logger_name=__name__, logger_path=log_path) + datetime = get_datetime_local() + time_str = get_str_from_datetime(datetime) + log_path = os.path.join('logs', f'{filename}_{time_str}.txt') + logger = setup_logger( + logger_name=filename, + logger_path=log_path, + file_level=logging.DEBUG, + console_level=logging.INFO) else: - logger = setup_logger(logger_name=__name__) + logger = setup_logger(logger_name=filename) # check input path exist_result = check_path_existence(args.smc_path, 'file') if exist_result != Existence.FileExist: diff --git a/tools/run_mview_sperson_estimator.py b/tools/run_mview_sperson_estimator.py new file mode 100644 index 00000000..f6862952 --- /dev/null +++ b/tools/run_mview_sperson_estimator.py @@ -0,0 +1,232 @@ +# yapf: disable +import argparse +import glob +import mmcv +import os +from xrprimer.data_structure.camera import ( + FisheyeCameraParameter, PinholeCameraParameter, +) +from xrprimer.utils.log_utils import logging, setup_logger +from xrprimer.utils.path_utils import Existence, check_path_existence +from xrprimer.visualization.keypoints.visualize_keypoints3d import ( + visualize_keypoints3d, +) + +from xrmocap.core.estimation.builder import build_estimator +from xrmocap.data_structure.body_model import SMPLXData +from xrmocap.utils.date_utils import get_datetime_local, get_str_from_datetime +from xrmocap.visualization.visualize_keypoints3d import ( + visualize_keypoints3d_projected, +) +from xrmocap.visualization.visualize_smpl import visualize_smpl_data + +# yapf: enable + + +def main(args): + filename = os.path.basename(__file__).split('.')[0] + # check input and output path + logger = setup_logger(logger_name=filename) + if check_path_existence(args.data_root) != \ + Existence.DirectoryExistNotEmpty or \ + check_path_existence(args.meta_path) != \ + Existence.DirectoryExistNotEmpty or \ + len(args.output_dir) <= 0: + logger.error('Not all necessary args have been configured.\n' + + f'data_root: {args.data_root}\n' + + f'meta_path: {args.meta_path}' + + f'output_dir: {args.output_dir}\n') + raise ValueError + if check_path_existence(args.output_dir) == Existence.MissingParent: + logger.error(f'Parent directory of {args.output_dir} does not exist.') + raise FileNotFoundError + elif check_path_existence(args.output_dir) == Existence.DirectoryNotExist: + os.mkdir(args.output_dir) + if check_path_existence('logs', 'dir') == Existence.DirectoryNotExist: + os.mkdir('logs') + if not args.disable_log_file: + datetime = get_datetime_local() + time_str = get_str_from_datetime(datetime) + log_path = os.path.join('logs', f'{filename}_{time_str}.txt') + logger = setup_logger( + logger_name=filename, + logger_path=log_path, + file_level=logging.DEBUG, + console_level=logging.INFO) + # build estimator + estimator_config = dict(mmcv.Config.fromfile(args.estimator_config)) + estimator_config['logger'] = logger + mview_sp_smpl_estimator = build_estimator(estimator_config) + scene_paths = sorted(glob.glob(os.path.join(args.meta_path, 'scene_*'))) + for _, scene_path in enumerate(scene_paths): + scene_name = os.path.basename(scene_path) + scene_output_dir = os.path.join(args.output_dir, scene_name) + # load input data + image_list_paths = sorted( + glob.glob(os.path.join(scene_path, 'image_list_view_*'))) + mview_img_list = [] + cam_param_list = [] + for _, img_list_path in enumerate(image_list_paths): + with open(img_list_path, 'r') as f_read: + lines = f_read.readlines() + view_idx = int( + os.path.basename(img_list_path).rsplit('.', + 1)[0].split('_')[-1]) + sview_img_list = [] + for line in lines: + rela_path = line.strip() + abs_path = os.path.join(args.data_root, rela_path) + sview_img_list.append(abs_path) + mview_img_list.append(sview_img_list) + cam_param_path = os.path.join( + scene_path, 'camera_parameters', + f'fisheye_param_{view_idx:02d}.json') + cam_param = FisheyeCameraParameter.fromfile(cam_param_path) + cam_param_list.append(cam_param) + # run estimation + keypoints2d_list, keypoints3d, smpl_data = mview_sp_smpl_estimator.run( + cam_param=cam_param_list, img_paths=mview_img_list) + # save results + keypoints2d_dir = os.path.join(scene_output_dir, 'keypoints2d_pred') + os.makedirs(keypoints2d_dir, exist_ok=True) + for keypoints2d, img_list_path in zip(keypoints2d_list, + image_list_paths): + view_idx = int( + os.path.basename(img_list_path).rsplit('.', + 1)[0].split('_')[-1]) + keypoints2d_path = os.path.join( + keypoints2d_dir, f'keypoints2d_view{view_idx:02d}.npz') + if keypoints2d is not None: + keypoints2d.dump(keypoints2d_path) + else: + logger.warning( + f'No keypoints2d has been detected in view{view_idx:02d}.') + keypoints3d_path = os.path.join(scene_output_dir, + 'keypoints3d_pred.npz') + keypoints3d.dump(keypoints3d_path) + if isinstance(smpl_data, SMPLXData): + smpl_type = 'smplx' + else: + smpl_type = 'smpl' + smpl_path = os.path.join(scene_output_dir, f'{smpl_type}_data.npz') + smpl_data.dump(smpl_path) + if args.visualize: + vis_dir = os.path.join(args.output_dir, 'visualize') + scene_vis_dir = os.path.join(vis_dir, scene_name) + os.makedirs(scene_vis_dir, exist_ok=True) + if keypoints3d is not None: + # visualize keypoints in a 3D scene + logger.info('Visualizing keypoints3d.') + visualize_keypoints3d( + keypoints=keypoints3d, + output_path=os.path.join(scene_vis_dir, + 'keypoints3d_pred.mp4'), + overwrite=True, + plot_axis=True, + plot_points=True, + plot_lines=True, + disable_tqdm=False, + logger=logger) + for idx, img_list_path in enumerate(image_list_paths): + view_idx = int( + os.path.basename(img_list_path).rsplit( + '.', 1)[0].split('_')[-1]) + logger.info( + f'Visualizing keypoints3d from view {view_idx:02d}.') + # visualize projected keypoints3d + visualize_keypoints3d_projected( + keypoints=keypoints3d, + camera=cam_param_list[idx], + output_path=os.path.join( + scene_vis_dir, + f'keypoints3d_pred_projected_{view_idx:02d}.mp4'), + overwrite=True, + plot_points=True, + plot_lines=True, + background_img_list=mview_img_list[idx], + disable_tqdm=False, + logger=logger) + if smpl_data is not None: + body_model = mview_sp_smpl_estimator.smplify.body_model + for idx, img_list_path in enumerate(image_list_paths): + view_idx = int( + os.path.basename(img_list_path).rsplit( + '.', 1)[0].split('_')[-1]) + logger.info(f'Visualizing {smpl_type.upper()}' + + f' from view {view_idx:02d}.') + with open(img_list_path, 'r') as f_read: + lines = f_read.readlines() + img_path = lines[0].strip() + img_dir = os.path.join(args.data_root, + os.path.dirname(img_path)) + fisheye_param = cam_param_list[idx] + pinhole_param = PinholeCameraParameter( + K=fisheye_param.intrinsic, + R=fisheye_param.extrinsic_r, + T=fisheye_param.extrinsic_t, + height=fisheye_param.height, + width=fisheye_param.width, + convention=fisheye_param.convention, + world2cam=fisheye_param.world2cam, + logger=logger) + visualize_smpl_data( + smpl_data=smpl_data, + body_model=body_model, + cam_param=pinhole_param, + output_path=os.path.join( + scene_vis_dir, + f'{smpl_type}_data_projected_{view_idx:02d}.mp4'), + overwrite=True, + background_dir=img_dir, + disable_tqdm=False, + logger=logger, + device=body_model.betas.device) + + +def setup_parser(): + parser = argparse.ArgumentParser( + description='Run MultiViewSinglePersonSMPLEstimator on a dataset.') + # input args + parser.add_argument( + '--data_root', + type=str, + help='Path to the root directory of' + ' the mview sperson dataset.', + default='') + parser.add_argument( + '--meta_path', + type=str, + help='Path to the meta-data dir.' + + ' Camera parameters and image paths will be read from here.', + default='') + # output args + parser.add_argument( + '--output_dir', + type=str, + help='Path to the directory where the' + + ' estimation results and visualization results save.', + default='') + # model args + parser.add_argument( + '--estimator_config', + help='Config file for MultiViewSinglePersonSMPLEstimator.', + type=str, + default='configs/humman_mocap/mview_sperson_smpl_estimator.py') + # output args + parser.add_argument( + '--visualize', + action='store_true', + help='If checked, visualize result.', + default=False) + # log args + parser.add_argument( + '--disable_log_file', + action='store_true', + help='If checked, log will not be written as file.', + default=False) + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = setup_parser() + main(args) diff --git a/xrmocap/data/data_converter/builder.py b/xrmocap/data/data_converter/builder.py index 1ca3c98d..09b2f20f 100644 --- a/xrmocap/data/data_converter/builder.py +++ b/xrmocap/data/data_converter/builder.py @@ -3,6 +3,7 @@ from .base_data_converter import BaseDataCovnerter from .campus_data_converter import CampusDataCovnerter +from .humman_smc_data_converter import HummanSMCDataCovnerter from .panoptic_data_converter import PanopticDataCovnerter from .shelf_data_converter import ShelfDataCovnerter @@ -15,6 +16,8 @@ name='ShelfDataCovnerter', module=ShelfDataCovnerter) DATA_CONVERTERS.register_module( name='PanopticDataCovnerter', module=PanopticDataCovnerter) +DATA_CONVERTERS.register_module( + name='HummanSMCDataCovnerter', module=HummanSMCDataCovnerter) def build_data_converter(cfg) -> BaseDataCovnerter: diff --git a/xrmocap/data/data_converter/humman_smc_data_converter.py b/xrmocap/data/data_converter/humman_smc_data_converter.py new file mode 100644 index 00000000..e95947cf --- /dev/null +++ b/xrmocap/data/data_converter/humman_smc_data_converter.py @@ -0,0 +1,333 @@ +# yapf: disable +import logging +import numpy as np +import os +from typing import List, Union +from xrprimer.transform.image.color import rgb2bgr +from xrprimer.utils.ffmpeg_utils import array_to_images +from xrprimer.utils.path_utils import ( + Existence, check_path_existence, check_path_suffix, +) + +from xrmocap.data.data_visualization import MviewMpersonDataVisualization +from xrmocap.data_structure.smc_reader import SMCReader +from xrmocap.human_perception.builder import MMtrackDetector, build_detector +from xrmocap.io.camera import get_all_color_kinect_parameter_from_smc +from .base_data_converter import BaseDataCovnerter + +# yapf: enable + + +class HummanSMCDataCovnerter(BaseDataCovnerter): + + def __init__(self, + data_root: str, + bbox_detector: Union[dict, None], + kps2d_estimator: Union[dict, None], + view_idxs: Union[str, List[int]] = 'all', + batch_size: int = 500, + meta_path: str = 'xrmocap_meta', + dataset_name: str = 'humman_smc', + visualize: bool = False, + verbose: bool = True, + logger: Union[None, str, logging.Logger] = None) -> None: + """Create a dir at meta_path, and convert camera parameters and + detected bbox2d, kps2d into XRLab format. + + Args: + data_root (str): + Path to the directory with smc files. + bbox_detector (Union[dict, None]): + A human bbox_detector, or its config, or None. + If None, converting perception 2d will be skipped. + kps2d_estimator (Union[dict, None]): + A top-down kps2d estimator, or its config, or None. + If None, converting perception 2d will be skipped. + view_idxs (Union[str, List[int]], optional): + A list of selected view indexes in each scene, + like [[0, 1], [0, 2, 4]]. + Defaults to 'all', all views will be selected. + batch_size (int, optional): + How many frames are loaded at the same time. Defaults to 500. + meta_path (str, optional): + Path to the meta-data dir. Defaults to 'xrmocap_meta'. + dataset_name (str, optional): + Name of the dataset. + Defaults to 'humman_smc'. + visualize (bool, optional): + Whether to visualize perception2d data and + ground-truth 3d data. Defaults to False. + verbose (bool, optional): + Whether to print(logger.info) information during converting. + Defaults to True. + logger (Union[None, str, logging.Logger], optional): + Logger for logging. If None, root logger will be selected. + Defaults to None. + """ + super().__init__( + data_root=data_root, + meta_path=meta_path, + dataset_name=dataset_name, + verbose=verbose, + logger=logger) + self.batch_size = batch_size + # check how many smc files in the data_root + file_names = sorted(os.listdir(data_root)) + self.view_idxs = view_idxs + self.smc_paths = [] + for file_name in file_names: + file_path = os.path.join(data_root, file_name) + if check_path_suffix(file_path, '.smc'): + self.smc_paths.append(file_path) + if isinstance(bbox_detector, dict): + bbox_detector['logger'] = logger + self.bbox_detector = build_detector(bbox_detector) + else: + self.bbox_detector = bbox_detector + + if isinstance(kps2d_estimator, dict): + kps2d_estimator['logger'] = logger + self.kps2d_estimator = build_detector(kps2d_estimator) + else: + self.kps2d_estimator = kps2d_estimator + self.visualize = visualize + if self.visualize: + vis_percep2d = self.bbox_detector is not None and \ + self.kps2d_estimator is not None + self.visualization = MviewMpersonDataVisualization( + data_root=data_root, + meta_path=meta_path, + vis_percep2d=vis_percep2d, + vis_gt_kps3d=False, + output_dir=os.path.join(meta_path, 'visualize'), + verbose=verbose, + logger=logger) + + def run(self, overwrite: bool = False) -> None: + """Convert xrmocap meta data from source dataset. Scenes will be + created, and there are image lists, XRPrimer cameras, ground truth + keypoints3d, perception2d in each of the scene. If any of + bbox_detector, kps2d_estimator is None, skip converting perception2d. + If visualize is checked, perception2d and ground truth will be + visualized. + + Args: + overwrite (bool, optional): + Whether replace the files at + self.meta_path. + Defaults to False. + """ + BaseDataCovnerter.run(self, overwrite=overwrite) + with open(os.path.join(self.meta_path, 'dataset_name.txt'), + 'w') as f_write: + f_write.write(f'{self.dataset_name}') + for scene_idx, smc_path in enumerate(self.smc_paths): + smc_reader = SMCReader(file_path=smc_path) + scene_dir = os.path.join(self.meta_path, f'scene_{scene_idx}') + os.makedirs(scene_dir, exist_ok=True) + self.covnert_smc_frames(scene_idx, smc_reader) + self.covnert_image_list(scene_idx, smc_reader) + self.convert_cameras(scene_idx, smc_reader) + if self.bbox_detector is not None and \ + self.kps2d_estimator is not None: + self.convert_perception_2d(scene_idx, smc_reader) + if self.visualize: + self.visualization.run(overwrite=overwrite) + + def covnert_smc_frames(self, scene_idx: int, + smc_reader: SMCReader) -> None: + """Convert smc in source dataset to extracted jpg pictures. + + Args: + scene_idx (int): + Index of this scene. + smc_reader (SMCReader): + SMCReader of this scene. + """ + scene_dir = os.path.join(self.meta_path, f'scene_{scene_idx}') + smc_path = self.smc_paths[scene_idx] + smc_name = os.path.basename(smc_path) + if self.verbose: + self.logger.info('Extracting image relative path list' + + f' for scene {scene_idx}, file {smc_name}.') + n_view = smc_reader.num_kinects + cur_view_idxs = _get_cur_view_idxs( + idxs_setting=self.view_idxs, n_view=n_view, logger=self.logger) + for view_idx in cur_view_idxs: + output_folder = os.path.join(scene_dir, + f'images_view_{view_idx:02d}') + sv_img_array = smc_reader.get_kinect_color(kinect_id=view_idx) + sv_img_array = rgb2bgr(sv_img_array) + array_to_images( + image_array=sv_img_array, + output_folder=output_folder, + img_format='%06d.jpg', + logger=self.logger) + + def covnert_image_list(self, scene_idx: int, + smc_reader: SMCReader) -> None: + """Convert source data to image list text file. + + Args: + scene_idx (int): + Index of this scene. + smc_reader (SMCReader): + SMCReader of this scene. + """ + scene_dir = os.path.join(self.meta_path, f'scene_{scene_idx}') + smc_path = self.smc_paths[scene_idx] + smc_name = os.path.basename(smc_path) + if self.verbose: + self.logger.info('Converting image relative path list' + + f' for scene {scene_idx}, file {smc_name}.') + n_view = smc_reader.num_kinects + cur_view_idxs = _get_cur_view_idxs( + idxs_setting=self.view_idxs, n_view=n_view, logger=self.logger) + n_frame = smc_reader.kinect_num_frames + for view_idx in cur_view_idxs: + image_dir = os.path.join(scene_dir, f'images_view_{view_idx:02d}') + frame_list = [] + for frame_idx in range(n_frame): + file_name = f'{frame_idx:06d}.jpg' + abs_file_path = os.path.join(image_dir, file_name) + rela_file_path = os.path.relpath(abs_file_path, self.data_root) + if check_path_existence(abs_file_path, + 'file') == Existence.FileExist: + frame_list.append(rela_file_path + '\n') + else: + self.logger.error('Humman dataset is broken.' + + f' Missing file: {abs_file_path}') + raise FileNotFoundError + frame_list[-1] = frame_list[-1][:-1] + with open( + os.path.join(scene_dir, + f'image_list_view_{view_idx:02d}.txt'), + 'w') as f_write: + f_write.writelines(frame_list) + + def convert_cameras(self, scene_idx: int, smc_reader: SMCReader) -> None: + """Convert smc data to XRPrimer camera parameters. + + Args: + scene_idx (int): + Index of this scene. + smc_reader (SMCReader): + SMCReader of this scene. + """ + scene_dir = os.path.join(self.meta_path, f'scene_{scene_idx}') + smc_path = self.smc_paths[scene_idx] + smc_name = os.path.basename(smc_path) + if self.verbose: + self.logger.info('Converting camera parameters' + + f' for scene {scene_idx}, file {smc_name}.') + cam_param_list = get_all_color_kinect_parameter_from_smc( + smc_reader=smc_reader, align_floor=True, logger=self.logger) + cam_dir = os.path.join(scene_dir, 'camera_parameters') + os.makedirs(cam_dir, exist_ok=True) + cur_view_idxs = _get_cur_view_idxs( + idxs_setting=self.view_idxs, + n_view=len(cam_param_list), + logger=self.logger) + for view_idx in cur_view_idxs: + cam_param = cam_param_list[view_idx] + cam_param.name = f'fisheye_param_{view_idx:02d}' + cam_param.dump(os.path.join(cam_dir, f'{cam_param.name}.json')) + + def convert_ground_truth(self, scene_idx: int, + smc_reader: SMCReader) -> None: + """SMC has no GT inside, create an empty array. + + Args: + scene_idx (int): + Index of this scene. + smc_reader (SMCReader): + SMCReader of this scene. + """ + if self.verbose: + self.logger.info( + f'Converting ground truth keypoints3d for scene {scene_idx}.') + raise NotImplementedError + + def convert_perception_2d(self, scene_idx: int, + smc_reader: SMCReader) -> None: + """Convert source data to 2D perception data, bbox + keypoints2d + + track info. + + Args: + scene_idx (int): + Index of this scene. + smc_reader (SMCReader): + SMCReader of this scene. + """ + scene_dir = os.path.join(self.meta_path, f'scene_{scene_idx}') + scene_bbox_list = [] + scene_keypoints_list = [] + n_view = smc_reader.num_kinects + cur_view_idxs = _get_cur_view_idxs( + idxs_setting=self.view_idxs, n_view=n_view, logger=self.logger) + for view_idx in cur_view_idxs: + if self.verbose: + self.logger.info('Inferring perception 2D data for' + + f' scene {scene_idx} view {view_idx}') + list_path = os.path.join(scene_dir, + f'image_list_view_{view_idx:02d}.txt') + with open(list_path, 'r') as f_read: + rela_path_list = f_read.readlines() + frame_list = [ + os.path.join(self.data_root, rela_path.strip()) + for rela_path in rela_path_list + ] + bbox_list = self.bbox_detector.infer_frames( + frame_path_list=frame_list, + disable_tqdm=not self.verbose, + multi_person=True, + load_batch_size=self.batch_size) + kps2d_list, _, _ = self.kps2d_estimator.infer_frames( + frame_path_list=frame_list, + bbox_list=bbox_list, + disable_tqdm=not self.verbose, + load_batch_size=self.batch_size) + keypoints2d = self.kps2d_estimator.get_keypoints_from_result( + kps2d_list) + n_person_max = 0 + n_frame = len(bbox_list) + for frame_idx, frame_bboxes in enumerate(bbox_list): + if n_person_max < len(frame_bboxes): + n_person_max = len(frame_bboxes) + bbox_arr = np.zeros(shape=(n_frame, n_person_max, 5)) + for frame_idx, frame_bboxes in enumerate(bbox_list): + for person_idx, person_bbox in enumerate(frame_bboxes): + if person_bbox is not None: + bbox_arr[frame_idx, person_idx] = person_bbox + scene_bbox_list.append(bbox_arr) + scene_keypoints_list.append(keypoints2d) + dict_to_save = dict( + bbox_tracked=isinstance(self.bbox_detector, MMtrackDetector), + bbox_convention='xyxy', + kps2d_convention=keypoints2d.get_convention(), + ) + for view_idx in range(self.n_view): + dict_to_save[f'bbox2d_view_{view_idx:02d}'] = scene_bbox_list[ + view_idx] + dict_to_save[f'kps2d_view_{view_idx:02d}'] = scene_keypoints_list[ + view_idx].get_keypoints() + dict_to_save[ + f'kps2d_mask_view_{view_idx:02d}'] = scene_keypoints_list[ + view_idx].get_mask() + np.savez_compressed( + file=os.path.join(scene_dir, 'perception_2d.npz'), **dict_to_save) + + +def _get_cur_view_idxs(idxs_setting: Union[str, List[int]], n_view: int, + logger: logging.Logger) -> List[int]: + if idxs_setting == 'all': + cur_view_idxs = [i for i in range(n_view)] + else: + for idx in idxs_setting: + if idx >= n_view: + logger.error( + f'View index {idx} is larger than the number of views' + + f' {n_view}.') + raise ValueError + cur_view_idxs = idxs_setting.copy() + return cur_view_idxs diff --git a/xrmocap/data/data_converter/panoptic_data_converter.py b/xrmocap/data/data_converter/panoptic_data_converter.py index fecc92c1..2631c07c 100644 --- a/xrmocap/data/data_converter/panoptic_data_converter.py +++ b/xrmocap/data/data_converter/panoptic_data_converter.py @@ -52,15 +52,15 @@ def __init__(self, A top-down kps2d estimator, or its config, or None. If None, converting perception 2d will be skipped. Defaults to None. - scene_names (List[str], optional): + scene_names (Union[str, List[str]], optional): A list of scene directory names, like ['160422_haggling1', '160906_ian5']. Defaults to 'all', all scenes in data_root will be selected. - view_idxs (List[int], optional): + view_idxs (Union[str, List[int]], optional): A list of selected view indexes in each scene, like [[0, 1], [0, 2, 4]]. Defaults to 'all', all views will be selected. - scene_range (List[List[int]], optional): + scene_range (Union[str, List[List[int]]], optional): Frame range of scenes. For instance, [[350, 470], [650, 750]] will select 120 frames from scene_0 and diff --git a/xrmocap/utils/date_utils.py b/xrmocap/utils/date_utils.py new file mode 100644 index 00000000..cb2c7214 --- /dev/null +++ b/xrmocap/utils/date_utils.py @@ -0,0 +1,35 @@ +from datetime import datetime +from dateutil import tz + + +def get_datetime_local() -> datetime: + """Get datetime in local time zone. + + Returns: + datetime: + An instance of datetime + in local time zone. + """ + datetime_src = datetime.now() + # Auto-detect zones: + # zone_src = tz.tzutc() + zone_dst = tz.tzlocal() + # # Tell the datetime object that it's in UTC time zone + # datetime_src = datetime_src.replace(tzinfo=zone_src) + # Convert time zone + datetime_dst = datetime_src.astimezone(zone_dst) + return datetime_dst + + +def get_str_from_datetime(datetime_instance: datetime, + format: str = '%Y.%m.%d_%H:%M:%S') -> str: + """Get string from datetime instance. + + Args: + datetime_instance (datetime): + An instance of datetime. + format (str): + Format of the string. + Defaults to '%Y.%m.%d_%H:%M:%S'. + """ + return datetime_instance.strftime(format)