Skip to content

log

Module diagram

classDiagram
  class log {
  }
  class dashboard {
  }
  class log_handler {
  }
  class logger {
  }
  class string_formatter {
  }
  log_handler --> dashboard
  log_handler --> logger

log

dashboard

log_handler

LogHandler 

LogHandler(name: str = 'SaigeResearch', logdir: Optional[str] = None, add_time_to_logdir: bool = False, logfile: Optional[str] = None, stdout: bool = False, level: str = 'INFO', formatter: str = f'%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s', dashboard: Optional[Dict] = None)
Source code in SaigeToolkit/log/log_handler.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def __init__(
    self,
    name: str = "SaigeResearch",
    logdir: Optional[str] = None,
    add_time_to_logdir: bool = False,
    logfile: Optional[str] = None,
    stdout: bool = False,
    level: str = "INFO",
    formatter: str = f"%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s",
    dashboard: Optional[Dict] = None,
):
    logfile_path = None
    if logdir is not None:
        if add_time_to_logdir:
            num_trial = 10
            for trial_index in range(num_trial):
                cur_logdir = os.path.join(logdir, self.get_time_string())
                try:
                    os.makedirs(cur_logdir, exist_ok=False)
                except FileExistsError:
                    if trial_index == (num_trial - 1):
                        traceback.print_exc()
                        sys.exit()
                    time.sleep(1.1)
                    continue
                else:
                    logdir = cur_logdir
                    break
        else:
            os.makedirs(logdir, exist_ok=False)

        if logfile is not None:
            logfile_path = os.path.join(logdir, logfile)

    self.logdir = logdir

    self.logger = configure_logger(
        name=name,
        level=level,
        formatter=formatter,
        path=logfile_path,
        stdout=stdout,
    )

    dashboard = dashboard or {}
    self.dashboard = build_dashboard(logdir=self.logdir, **dashboard)

    self.logger.info(f"{type(self).__name__} logdir {self.logdir}")
    self.logger.info(
        f"{type(self).__name__} dashboard {'ENABLED' if self.dashboard.enabled else 'DISABLED'}"
    )
skip_if_logdir_is_none 
skip_if_logdir_is_none(method)

Decorator to skip the method if logdir is None.

Source code in SaigeToolkit/log/log_handler.py 
80
81
82
83
84
85
86
87
88
89
90
91
def skip_if_logdir_is_none(method):
    """Decorator to skip the method if logdir is None."""

    @wraps(method)
    def decorator(self, *args, **kwargs):
        if getattr(self, "logdir", None) is None:
            if getattr(self, "logger", None) is not None:
                self.logger.info(f"{type(self).__name__} {method.__name__} skipped: logdir is None")
            return
        return method(self, *args, **kwargs)

    return decorator
save_json 
save_json(data: Dict, json_path: str = 'log.json') -> None

Save the data in the {self.logdir}/{json_path}.

Parameters:

  • data (Dict) –

    data to save

  • json_path (str, default: 'log.json' ) –

    json path. Defaults to "log.json".

Source code in SaigeToolkit/log/log_handler.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@skip_if_logdir_is_none
def save_json(self, data: Dict, json_path: str = "log.json") -> None:
    """Save the data in the {self.logdir}/{json_path}.

    Args:
        data (Dict): data to save
        json_path (str, optional): json path. Defaults to "log.json".
    """
    save_path = os.path.join(self.logdir, json_path)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)
    with open(save_path, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4, sort_keys=False)
save_image 
save_image(image: Image, file_path: str) -> None

Save the image in the {self.logdir}/{file_path}.

Source code in SaigeToolkit/log/log_handler.py 
106
107
108
109
110
111
@skip_if_logdir_is_none
def save_image(self, image: Image.Image, file_path: str) -> None:
    """Save the image in the {self.logdir}/{file_path}."""
    save_path = os.path.join(self.logdir, file_path)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)
    image.save(save_path)
save_code 
save_code(module: str, directory: str = 'source_code') -> None

Save the module codes in the "source_code" directory.

Parameters:

  • target (str) –

    module path

  • directory (str, default: 'source_code' ) –

    save directory. Defaults to "source_code".

Usage 
import __main__

log_handler.save_code(__main__.__file__)
Source code in SaigeToolkit/log/log_handler.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
@skip_if_logdir_is_none
def save_code(self, module: str, directory: str = "source_code") -> None:
    """Save the module codes in the "source_code" directory.

    Args:
        target (str): module path
        directory (str, optional): save directory. Defaults to "source_code".

    Usage:
        ```python

        import __main__

        log_handler.save_code(__main__.__file__)
        ```
    """
    code_save_dir = os.path.join(self.logdir, directory)
    code_original_dir = os.path.dirname(module)

    module_list = sorted(parse_compile_list(module))
    code_path_list = [os.path.basename(module)]
    for module in module_list:
        code_path = module.replace(".", os.path.sep) + ".py"
        if os.path.isfile(code_path):
            code_path_list.append(code_path)

    for code_path in code_path_list:
        code_original_path = os.path.join(code_original_dir, code_path)
        code_save_path = os.path.join(code_save_dir, code_path)
        os.makedirs(os.path.dirname(code_save_path), exist_ok=True)
        shutil.copy(code_original_path, code_save_path)

    self.logger.info(f"{type(self).__name__} All codes are saved in {code_save_dir}.")
save_yaml 
save_yaml(data: Dict, filename: str) -> None

Save the dict in yaml format.

Parameters:

  • data (Dict) –

    Dict to save

  • filename (str) –

    filename

Source code in SaigeToolkit/log/log_handler.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@skip_if_logdir_is_none
def save_yaml(self, data: Dict, filename: str) -> None:
    """Save the dict in yaml format.

    Args:
        data (Dict): Dict to save
        filename (str): filename
    """
    save_path = os.path.join(self.logdir, filename)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)

    OmegaConf.save(data, save_path)

    self.logger.info(f"{type(self).__name__} yaml file is saved in {save_path}.")
save_source_info 
save_source_info(git_path: str) -> None

Save current time (KST) + git info + run command in "source_info.json".

Parameters:

  • git_path (str) –

    git directory path.

Source code in SaigeToolkit/log/log_handler.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
@skip_if_logdir_is_none
def save_source_info(self, git_path: str) -> None:
    """Save current time (KST) + git info + run command in "source_info.json".

    Args:
        git_path (str): git directory path.
    """
    save_dir = self.logdir
    filename = "source_info.json"

    save_path = os.path.join(save_dir, filename)
    source_info = get_source_info(git_path)
    self.save_json(source_info, filename)

    self.logger.info(f"{type(self).__name__} source info is saved in {save_path}.")
save_checkpoint 
save_checkpoint(trainer: BaseTrainer, filename: str, password: Optional[str] = None) -> None

Save checkpoint in the "checkpoint" directory.

Parameters:

  • trainer (Trainer) –

    An instance containing information to be saved in the checkpoint.

  • filename (str) –

    filename

  • password (Optional[str], default: None ) –

    Password for checkpoint file. Defaults to None.

Source code in SaigeToolkit/log/log_handler.py 
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
@skip_if_logdir_is_none
def save_checkpoint(
    self, trainer: BaseTrainer, filename: str, password: Optional[str] = None
) -> None:
    """Save checkpoint in the "checkpoint" directory.

    Args:
        trainer (Trainer): An instance containing information to be saved in the checkpoint.
        filename (str): filename
        password (Optional[str], optional): Password for checkpoint file. Defaults to None.
    """
    save_dir = os.path.join(self.logdir, "checkpoint")
    save_path = os.path.join(save_dir, filename)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)

    trainer.save_checkpoint(save_path, password=password)

logger

string_formatter

list_to_pretty_string 

list_to_pretty_string(list_of_number: List[Union[int, float]], row_name: str = None, each_length: int = 10, indent: int = 12) -> str

list of Union[int, float] string으로 바꾸는 함수입니다. each_length의 길이가 되도록 " "를 늘려줍니다. log 예시: 

     class 0|         0|         1|         2|
Args: list_of_number (List[Union[int, float]]): 숫자가 들어있는 list입니다. row_name (str, optional): 각 row의 이름을 넣어줍니다. Defaults to None. each_length (int, optional): 각 원소마다 가지는 길이. Defaults to 10. indent (int, optional): indentation. Defaults to 12.

Returns: str

Source code in SaigeToolkit/log/string_formatter.py 
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
def list_to_pretty_string(
    list_of_number: List[Union[int, float]],
    row_name: str = None,
    each_length: int = 10,
    indent: int = 12,
) -> str:
    """
    list of Union[int, float] string으로 바꾸는 함수입니다.
    each_length의 길이가 되도록 " "를 늘려줍니다.
    log 예시:
    ```
         class 0|         0|         1|         2|
    ```
    Args:
        list_of_number (List[Union[int, float]]): 숫자가 들어있는 list입니다.
        row_name (str, optional): 각 row의 이름을 넣어줍니다. Defaults to None.
        each_length (int, optional): 각 원소마다 가지는 길이. Defaults to 10.
        indent (int, optional): indentation. Defaults to 12.

    Returns: str
    """

    if row_name is None:
        row_name = ""
    stdout = "\n" + row_name.rjust(indent, " ") + "|"
    for integer in list_of_number:
        stdout += str(integer).rjust(each_length, " ") + "|"
    return stdout

confusion_matrix_to_pretty_string 

confusion_matrix_to_pretty_string(confusion_matrix: List[List[Union[int, float]]], each_length: int = 10, indent: int = 12) -> str

confusion matrix를 logging하기 위해 string으로 바꾸는 함수입니다. each_length의 길이가 되도록 " "를 늘려줍니다. log 예시: 

...
2023-xx-xx xx:yy:zz,000 INFO [train.py:143]
    object/confusion_matrix:
               |   class 0|   class 1|   class 2|
        class 0|         0|       132|       123|
        class 1|         1|       458|      8394|
        class 2|        10|         1|         2|
    object/accuracy: 0.0000
    ...
Args: confusion_matrix (List[List[Union[int, float]]]): confusion matrix입니다. each_length (int, optional): 각 원소마다 가지는 길이. Defaults to 10. indent (int, optional): indentation. Defaults to 12.

Returns: str

Source code in SaigeToolkit/log/string_formatter.py 
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def confusion_matrix_to_pretty_string(
    confusion_matrix: List[List[Union[int, float]]],
    each_length: int = 10,
    indent: int = 12,
) -> str:
    """
    confusion matrix를 logging하기 위해 string으로 바꾸는 함수입니다.
    each_length의 길이가 되도록 " "를 늘려줍니다.
    log 예시:
    ```
    ...
    2023-xx-xx xx:yy:zz,000 INFO [train.py:143]
        object/confusion_matrix:
                   |   class 0|   class 1|   class 2|
            class 0|         0|       132|       123|
            class 1|         1|       458|      8394|
            class 2|        10|         1|         2|
        object/accuracy: 0.0000
        ...
    ```
    Args:
        confusion_matrix (List[List[Union[int, float]]]): confusion matrix입니다.
        each_length (int, optional): 각 원소마다 가지는 길이. Defaults to 10.
        indent (int, optional): indentation. Defaults to 12.

    Returns: str
    """

    stdout = "\n" + " " * indent + "|"
    assert len(confusion_matrix) > 0, "confusion matrix is empty"
    nclasses = len(confusion_matrix[0])
    for idx in range(nclasses):
        stdout += f"class {idx}".rjust(each_length, " ") + "|"
    for i, row in enumerate(confusion_matrix):
        row_name = f"class {i}"
        stdout += list_to_pretty_string(
            list_of_number=row,
            row_name=row_name,
            each_length=each_length,
            indent=indent,
        )
    return stdout