diff --git a/mmengine/config/config.py b/mmengine/config/config.py index b34b606cfb..ddc246655a 100644 --- a/mmengine/config/config.py +++ b/mmengine/config/config.py @@ -40,6 +40,18 @@ import re # type: ignore +def _lazy2string(cfg_dict, dict_type=None): + if isinstance(cfg_dict, dict): + dict_type = dict_type or type(cfg_dict) + return dict_type({k: _lazy2string(v) for k, v in dict.items(cfg_dict)}) + elif isinstance(cfg_dict, (tuple, list)): + return type(cfg_dict)(_lazy2string(v) for v in cfg_dict) + elif isinstance(cfg_dict, (LazyAttr, LazyObject)): + return f'{cfg_dict.module}.{str(cfg_dict)}' + else: + return cfg_dict + + class ConfigDict(Dict): """A dictionary for config which has the same interface as python's built- in dictionary and can be used as a normal dictionary. @@ -249,7 +261,7 @@ def _merge_a_into_b(a, b): for key, value in merged.items(): self[key] = value - def to_dict(self): + def _to_lazy_dict(self): """Convert the ConfigDict to a normal dictionary recursively, and keep the ``LazyObject`` or ``LazyAttr`` object not built.""" @@ -268,6 +280,11 @@ def _to_dict(data): return _to_dict(self) + def to_dict(self): + """Convert the ConfigDict to a normal dictionary recursively, and keep + the ``LazyObject`` or ``LazyAttr`` object not built.""" + return _lazy2string(self, dict_type=dict) + def add_args(parser: ArgumentParser, cfg: dict, @@ -318,6 +335,8 @@ class Config: cfg_text (str, optional): Text of config. Defaults to None. filename (str or Path, optional): Name of config file. Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. Here is a simple example: @@ -348,7 +367,8 @@ def __init__(self, cfg_dict: dict = None, cfg_text: Optional[str] = None, filename: Optional[Union[str, Path]] = None, - env_variables: Optional[dict] = None): + env_variables: Optional[dict] = None, + format_python_code: bool = True): filename = str(filename) if isinstance(filename, Path) else filename if cfg_dict is None: cfg_dict = dict() @@ -363,6 +383,7 @@ def __init__(self, cfg_dict = ConfigDict(cfg_dict) super().__setattr__('_cfg_dict', cfg_dict) super().__setattr__('_filename', filename) + super().__setattr__('_format_python_code', format_python_code) if cfg_text: text = cfg_text elif filename: @@ -380,7 +401,8 @@ def fromfile(filename: Union[str, Path], use_predefined_variables: bool = True, import_custom_modules: bool = True, use_environment_variables: bool = True, - lazy_import: Optional[bool] = None) -> 'Config': + lazy_import: Optional[bool] = None, + format_python_code: bool = True) -> 'Config': """Build a Config instance from config file. Args: @@ -392,6 +414,8 @@ def fromfile(filename: Union[str, Path], lazy_import (bool): Whether to load config in `lazy_import` mode. If it is `None`, it will be deduced by the content of the config file. Defaults to None. + format_python_code (bool): Whether to format Python code by yapf. + Defaults to True. Returns: Config: Config instance built from config file. @@ -434,13 +458,18 @@ def fromfile(filename: Union[str, Path], raise e finally: ConfigDict.lazy = False - for key, value in list(cfg_dict.to_dict().items()): + + # delete builtin imported objects + for key, value in list(cfg_dict._to_lazy_dict().items()): if isinstance(value, (types.FunctionType, types.ModuleType)): cfg_dict.pop(key) # disable lazy import to get the real type. See more details about # lazy in the docstring of ConfigDict - cfg = Config(cfg_dict, filename=filename) + cfg = Config( + cfg_dict, + filename=filename, + format_python_code=format_python_code) object.__setattr__(cfg, '_imported_names', imported_names) return cfg @@ -1321,8 +1350,6 @@ def _indent(s_, num_spaces): def _format_basic_types(k, v, use_mapping=False): if isinstance(v, str): v_str = repr(v) - elif isinstance(v, (LazyObject, LazyAttr)): - v_str = f"'{v.module}.{str(v)}'" else: v_str = str(v) @@ -1354,8 +1381,6 @@ def _format_list_tuple(k, v, use_mapping=False): v_str += f'{_indent(_format_list_tuple(None, item), indent)},\n' # noqa: 501 elif isinstance(item, str): v_str += f'{_indent(repr(item), indent)},\n' - elif isinstance(item, (LazyObject, LazyAttr)): - v_str += f"'{str(item)}',\n" else: v_str += str(item) + ',\n' if k is None: @@ -1385,9 +1410,7 @@ def _format_dict(input_dict, outest_level=False): for idx, (k, v) in enumerate(input_dict.items()): is_last = idx >= len(input_dict) - 1 end = '' if outest_level or is_last else ',' - if isinstance(v, (LazyObject, LazyAttr)): - attr_str = _format_basic_types(k, v, use_mapping) + end - elif isinstance(v, dict): + if isinstance(v, dict): v_str = '\n' + _format_dict(v) if use_mapping: k_str = f"'{k}'" if isinstance(k, str) else str(k) @@ -1406,19 +1429,20 @@ def _format_dict(input_dict, outest_level=False): r += '}' return r - cfg_dict = self._to_lazy_dict() + cfg_dict = self.to_dict() text = _format_dict(cfg_dict, outest_level=True) - # copied from setup.cfg - yapf_style = dict( - based_on_style='pep8', - blank_line_before_nested_class_or_def=True, - split_before_expression_after_opening_paren=True) - try: - text, _ = FormatCode(text, style_config=yapf_style, verify=True) - except: # noqa: E722 - raise SyntaxError('Failed to format the config file, please ' - f'check the syntax of: \n{text}') - + if self._format_python_code: + # copied from setup.cfg + yapf_style = dict( + based_on_style='pep8', + blank_line_before_nested_class_or_def=True, + split_before_expression_after_opening_paren=True) + try: + text, _ = FormatCode( + text, style_config=yapf_style, verify=True) + except: # noqa: E722 + raise SyntaxError('Failed to format the config file, please ' + f'check the syntax of: \n{text}') return text def __repr__(self): @@ -1490,7 +1514,7 @@ def dump(self, file: Optional[Union[str, Path]] = None): str or None: Config text. """ file = str(file) if isinstance(file, Path) else file - cfg_dict = super().__getattribute__('_cfg_dict').to_dict() + cfg_dict = self.to_dict() if file is None: if self.filename is None or self.filename.endswith('.py'): return self.pretty_text @@ -1594,7 +1618,7 @@ def _is_lazy_import(filename: str) -> bool: def _to_lazy_dict(self, keep_imported: bool = False) -> dict: """Convert config object to dictionary and filter the imported object.""" - res = self._cfg_dict.to_dict() + res = self._cfg_dict._to_lazy_dict() if hasattr(self, '_imported_names') and not keep_imported: res = { key: value @@ -1613,21 +1637,7 @@ def to_dict(self, keep_imported: bool = False): If you import third-party objects in the config file, all imported objects will be converted to a string like ``torch.optim.SGD`` """ - _cfg_dict = self._to_lazy_dict(keep_imported) - - def lazy2string(cfg_dict): - if isinstance(cfg_dict, dict): - return type(cfg_dict)( - {k: lazy2string(v) - for k, v in cfg_dict.items()}) - elif isinstance(cfg_dict, (tuple, list)): - return type(cfg_dict)(lazy2string(v) for v in cfg_dict) - elif isinstance(cfg_dict, (LazyAttr, LazyObject)): - return f'{cfg_dict.module}.{str(cfg_dict)}' - else: - return cfg_dict - - return lazy2string(_cfg_dict) + return self._cfg_dict.to_dict() class DictAction(Action): diff --git a/tests/test_config/test_config.py b/tests/test_config/test_config.py index 3e40975555..2e97b0afa7 100644 --- a/tests/test_config/test_config.py +++ b/tests/test_config/test_config.py @@ -1085,7 +1085,7 @@ def test_build_lazy(self): ])) cfg_dict = ConfigDict(raw) # check `items` and values - self.assertDictEqual(cfg_dict.to_dict(), raw) + self.assertDictEqual(cfg_dict._to_lazy_dict(), raw) self._check(cfg_dict) # check getattr @@ -1132,10 +1132,10 @@ def test_build_lazy(self): def _check(self, cfg_dict): self._recursive_check_lazy(cfg_dict, lambda x: not isinstance(x, LazyObject)) - self._recursive_check_lazy(cfg_dict.to_dict(), + self._recursive_check_lazy(cfg_dict._to_lazy_dict(), lambda x: x is not mmengine) self._recursive_check_lazy( - cfg_dict.to_dict(), lambda x: not isinstance(x, ConfigDict) + cfg_dict._to_lazy_dict(), lambda x: not isinstance(x, ConfigDict) if isinstance(x, dict) else True) self._recursive_check_lazy( cfg_dict, lambda x: isinstance(x, ConfigDict)