Skip to content

config

util.config

read_yml 

read_yml(config: str) -> Dict[str, Any]

load config dictionary

Parameters:

  • config (str) –

    file path

Returns:

  • Dict[str, Any] –

    Dict[str, Any]: result config dict

Source code in SaigeToolkit/util/config.py 
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
def read_yml(config: str) -> Dict[str, Any]:
    """load config dictionary

    Args:
        config (str): file path

    Returns:
        Dict[str, Any]: result config dict
    """
    cfg = _load_yml(config)

    cfg = update_import(cfg)

    return cfg

_load_yml 

_load_yml(file: str) -> Dict[str, Any]

load config dictionary from *.yml file

Source code in SaigeToolkit/util/config.py 
24
25
26
27
28
29
30
31
32
def _load_yml(file: str) -> Dict[str, Any]:
    """load config dictionary from *.yml file"""

    assert os.path.isfile(file), f"string type data config should direct a file {file}"

    with open(file, encoding="utf-8") as fp:
        cfg = yaml.load(fp, Loader=yaml.FullLoader)

    return cfg

update_import 

update_import(cfg: Dict[str, Any]) -> Dict[str, Any]

recursively import and override base config, and there is two conditions of importing:

  • if dict has import key and its value pointing another config file (*.yml)
  • if dict value is pointing another config file (*.yml)

Parameters:

  • cfg (Dict[str, Any]) –

    base config

Returns:

  • Dict[str, Any] –

    Dict[str, Any]: result config (import conditions updated)

Source code in SaigeToolkit/util/config.py 
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
def update_import(cfg: Dict[str, Any]) -> Dict[str, Any]:
    """recursively import and override base config,
    and there is two conditions of importing:

    - if dict has `import` key and its value pointing another config file (*.yml)
    - if dict value is pointing another config file (*.yml)

    Args:
        cfg (Dict[str, Any]): base config

    Returns:
        Dict[str, Any]: result config (import conditions updated)
    """

    # 1. has `import` key and its value pointing another config file
    cfg_import = cfg.pop("import", None)
    if cfg_import:
        # update_import function is called for recursive update
        cfg_mother = update_import(_load_yml(cfg_import))
        cfg = override_dict(cfg_mother, cfg)

    for k, v in cfg.items():
        # if there's no import and value is dict type, update recursively
        if isinstance(v, dict):
            cfg[k] = update_import(v)

        # 2. value is pointing another config file (*.yml)
        elif isinstance(v, str) and v.endswith(".yml"):
            # update_import function is called for recursive update
            cfg[k] = update_import(_load_yml(v))

    return cfg

save_yml 

save_yml(config: Dict[str, Any], path: str)
Source code in SaigeToolkit/util/config.py 
69
70
71
72
73
74
75
76
77
78
79
def save_yml(config: Dict[str, Any], path: str):
    yaml.add_representer(
        list,
        lambda dumper, value: dumper.represent_sequence("tag:yaml.org,2002:seq", value, flow_style=True),
    )
    yaml.add_representer(
        type(None), lambda dumper, value: dumper.represent_scalar("tag:yaml.org,2002:null", "")
    )

    with open(path, "w") as f:
        yaml.dump(config, f, sort_keys=False, indent=4, allow_unicode=True, width=1000)

override_dict 

override_dict(target: Dict, source: Dict) -> Dict

recursively override dictionaries

Source code in SaigeToolkit/util/config.py 
82
83
84
85
86
87
88
89
90
91
def override_dict(target: Dict, source: Dict) -> Dict:
    """recursively override dictionaries"""
    if not source:
        return target
    for k, v in source.items():
        if k in target and isinstance(target[k], Dict) and isinstance(v, dict):
            target[k] = override_dict(target[k], v)
        else:
            target[k] = v
    return target

convert_keys_string_to_list 

convert_keys_string_to_list(function)

keys 입력값이 str인 경우 . 으로 분할해 함수에 전달해줍니다.

Source code in SaigeToolkit/util/config.py 
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
def convert_keys_string_to_list(function):
    """
    keys 입력값이 str인 경우 `.` 으로 분할해 함수에 전달해줍니다.
    """

    @wraps(function)
    def wrapper(obj, keys, *args, **kwargs):
        if isinstance(keys, str):
            keys = keys.split(".")
        return function(obj, keys, *args, **kwargs)

    return wrapper

check_key_exists 

check_key_exists(obj: Union[Dict, Tuple, List], key: Any)
Source code in SaigeToolkit/util/config.py 
108
109
110
111
112
113
def check_key_exists(obj: Union[Dict, Tuple, List], key: Any):
    if isinstance(obj, Dict) and key not in obj:
        return False
    elif isinstance(obj, (Tuple, List)) and len(obj) < key + 1:
        return False
    return True

get_tree_node 

get_tree_node(obj: Any, keys: List) -> Any

tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 리턴, 최종 노드가 존재하지 않으면 error 발생

Source code in SaigeToolkit/util/config.py 
116
117
118
119
120
121
122
123
124
125
126
127
@convert_keys_string_to_list
def get_tree_node(obj: Any, keys: List) -> Any:
    """tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 리턴, 최종 노드가 존재하지 않으면 error 발생"""
    for key in keys:
        if isinstance(obj, (Tuple, List)) and isinstance(key, str):
            key = int(key)
        if not check_key_exists(obj=obj, key=key):
            raise KeyError
        if not isinstance(obj, (Tuple, Dict, List)):
            raise TypeError
        obj = obj[key]
    return obj

set_tree_node 

set_tree_node(obj: Any, keys: List, value: Any) -> Any

tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 수정, 최종 노드가 존재하지 않으면 error 발생

Source code in SaigeToolkit/util/config.py 
130
131
132
133
134
135
136
137
138
139
140
141
@convert_keys_string_to_list
def set_tree_node(obj: Any, keys: List, value: Any) -> Any:
    """tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 수정, 최종 노드가 존재하지 않으면 error 발생"""
    obj = get_tree_node(obj=obj, keys=keys[:-1])
    key = keys[-1]
    if isinstance(obj, List) and isinstance(key, str):
        key = int(key)
    if not check_key_exists(obj=obj, key=key):
        raise KeyError
    if not isinstance(obj, (Tuple, Dict, List)):
        raise TypeError
    obj[key] = value

get_tree_node_with_default 

get_tree_node_with_default(obj: Any, keys: List, value: Optional[Any] = None, ignore_type_error: bool = False) -> Any

tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 리턴, 최종 노드가 존재하지 않으면 value 리턴, ignore_type_error=True 일 경우 최종 value가 더이상 탐색하지 못할 때에도 error 대신 default value를 리턴.

Source code in SaigeToolkit/util/config.py 
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@convert_keys_string_to_list
def get_tree_node_with_default(
    obj: Any, keys: List, value: Optional[Any] = None, ignore_type_error: bool = False
) -> Any:
    """tree-like obj에 대해 keys의 값을 순서대로 각 레벨에서 탐색해 최종 노드의 값을 리턴, 최종 노드가 존재하지 않으면 value 리턴,
    ignore_type_error=True 일 경우 최종 value가 더이상 탐색하지 못할 때에도 error 대신 default value를 리턴.
    """
    try:
        obj = get_tree_node(obj, keys)
    except KeyError:
        obj = value
    except TypeError:
        if not ignore_type_error:
            raise TypeError
        obj = value

    return obj

flatten_tree 

flatten_tree(obj, parent_key='', sep='.', depth=0, max_depth=None) -> Dict

tree-like obj를 1 레벨 dictionary로 변환

Source code in SaigeToolkit/util/config.py 
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
def flatten_tree(obj, parent_key="", sep=".", depth=0, max_depth=None) -> Dict:
    """tree-like obj를 1 레벨 dictionary로 변환"""
    flattened = dict()
    if isinstance(obj, Dict):
        iterator = obj.items()
    else:
        iterator = enumerate(obj)

    for key, value in iterator:
        flattened_key = f"{parent_key}{sep}{key}" if parent_key else key
        if isinstance(value, (Dict, List, Tuple)) and (max_depth is None or depth < max_depth):
            nested = flatten_tree(
                value,
                parent_key=flattened_key,
                sep=sep,
                depth=depth + 1,
                max_depth=max_depth,
            )
        else:
            nested = {flattened_key: value}

        for flattened_key, flattened_value in nested.items():
            if flattened_key in flattened:
                raise KeyError(f"multiple {flattened_key} exist in tree")
            flattened[flattened_key] = flattened_value

    return flattened