Skip to content

log_handler

log.log_handler

LogHandler 

LogHandler(name: str = 'SaigeResearch', logdir: Optional[str] = None, add_time_to_logdir: bool = False, logfile: Optional[str] = None, stdout: bool = False, level: str = 'INFO', formatter: str = f'%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s', dashboard: Optional[Dict] = None)
Source code in SaigeToolkit/log/log_handler.py 
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
def __init__(
    self,
    name: str = "SaigeResearch",
    logdir: Optional[str] = None,
    add_time_to_logdir: bool = False,
    logfile: Optional[str] = None,
    stdout: bool = False,
    level: str = "INFO",
    formatter: str = f"%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s",
    dashboard: Optional[Dict] = None,
):
    logfile_path = None
    if logdir is not None:
        if add_time_to_logdir:
            num_trial = 10
            for trial_index in range(num_trial):
                cur_logdir = os.path.join(logdir, self.get_time_string())
                try:
                    os.makedirs(cur_logdir, exist_ok=False)
                except FileExistsError:
                    if trial_index == (num_trial - 1):
                        traceback.print_exc()
                        sys.exit()
                    time.sleep(1.1)
                    continue
                else:
                    logdir = cur_logdir
                    break
        else:
            os.makedirs(logdir, exist_ok=False)

        if logfile is not None:
            logfile_path = os.path.join(logdir, logfile)

    self.logdir = logdir

    self.logger = configure_logger(
        name=name,
        level=level,
        formatter=formatter,
        path=logfile_path,
        stdout=stdout,
    )

    dashboard = dashboard or {}
    self.dashboard = build_dashboard(logdir=self.logdir, **dashboard)

    self.logger.info(f"{type(self).__name__} logdir {self.logdir}")
    self.logger.info(
        f"{type(self).__name__} dashboard {'ENABLED' if self.dashboard.enabled else 'DISABLED'}"
    )

skip_if_logdir_is_none 

skip_if_logdir_is_none(method)

Decorator to skip the method if logdir is None.

Source code in SaigeToolkit/log/log_handler.py 
80
81
82
83
84
85
86
87
88
89
90
91
def skip_if_logdir_is_none(method):
    """Decorator to skip the method if logdir is None."""

    @wraps(method)
    def decorator(self, *args, **kwargs):
        if getattr(self, "logdir", None) is None:
            if getattr(self, "logger", None) is not None:
                self.logger.info(f"{type(self).__name__} {method.__name__} skipped: logdir is None")
            return
        return method(self, *args, **kwargs)

    return decorator

save_json 

save_json(data: Dict, json_path: str = 'log.json') -> None

Save the data in the {self.logdir}/{json_path}.

Parameters:

  • data (Dict) –

    data to save

  • json_path (str, default: 'log.json' ) –

    json path. Defaults to "log.json".

Source code in SaigeToolkit/log/log_handler.py 
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
@skip_if_logdir_is_none
def save_json(self, data: Dict, json_path: str = "log.json") -> None:
    """Save the data in the {self.logdir}/{json_path}.

    Args:
        data (Dict): data to save
        json_path (str, optional): json path. Defaults to "log.json".
    """
    save_path = os.path.join(self.logdir, json_path)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)
    with open(save_path, "w", encoding="utf-8") as f:
        json.dump(data, f, ensure_ascii=False, indent=4, sort_keys=False)

save_image 

save_image(image: Image, file_path: str) -> None

Save the image in the {self.logdir}/{file_path}.

Source code in SaigeToolkit/log/log_handler.py 
106
107
108
109
110
111
@skip_if_logdir_is_none
def save_image(self, image: Image.Image, file_path: str) -> None:
    """Save the image in the {self.logdir}/{file_path}."""
    save_path = os.path.join(self.logdir, file_path)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)
    image.save(save_path)

save_code 

save_code(module: str, directory: str = 'source_code') -> None

Save the module codes in the "source_code" directory.

Parameters:

  • target (str) –

    module path

  • directory (str, default: 'source_code' ) –

    save directory. Defaults to "source_code".

Usage 
import __main__

log_handler.save_code(__main__.__file__)
Source code in SaigeToolkit/log/log_handler.py 
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
@skip_if_logdir_is_none
def save_code(self, module: str, directory: str = "source_code") -> None:
    """Save the module codes in the "source_code" directory.

    Args:
        target (str): module path
        directory (str, optional): save directory. Defaults to "source_code".

    Usage:
        ```python

        import __main__

        log_handler.save_code(__main__.__file__)
        ```
    """
    code_save_dir = os.path.join(self.logdir, directory)
    code_original_dir = os.path.dirname(module)

    module_list = sorted(parse_compile_list(module))
    code_path_list = [os.path.basename(module)]
    for module in module_list:
        code_path = module.replace(".", os.path.sep) + ".py"
        if os.path.isfile(code_path):
            code_path_list.append(code_path)

    for code_path in code_path_list:
        code_original_path = os.path.join(code_original_dir, code_path)
        code_save_path = os.path.join(code_save_dir, code_path)
        os.makedirs(os.path.dirname(code_save_path), exist_ok=True)
        shutil.copy(code_original_path, code_save_path)

    self.logger.info(f"{type(self).__name__} All codes are saved in {code_save_dir}.")

save_yaml 

save_yaml(data: Dict, filename: str) -> None

Save the dict in yaml format.

Parameters:

  • data (Dict) –

    Dict to save

  • filename (str) –

    filename

Source code in SaigeToolkit/log/log_handler.py 
147
148
149
150
151
152
153
154
155
156
157
158
159
160
@skip_if_logdir_is_none
def save_yaml(self, data: Dict, filename: str) -> None:
    """Save the dict in yaml format.

    Args:
        data (Dict): Dict to save
        filename (str): filename
    """
    save_path = os.path.join(self.logdir, filename)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)

    OmegaConf.save(data, save_path)

    self.logger.info(f"{type(self).__name__} yaml file is saved in {save_path}.")

save_source_info 

save_source_info(git_path: str) -> None

Save current time (KST) + git info + run command in "source_info.json".

Parameters:

  • git_path (str) –

    git directory path.

Source code in SaigeToolkit/log/log_handler.py 
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
@skip_if_logdir_is_none
def save_source_info(self, git_path: str) -> None:
    """Save current time (KST) + git info + run command in "source_info.json".

    Args:
        git_path (str): git directory path.
    """
    save_dir = self.logdir
    filename = "source_info.json"

    save_path = os.path.join(save_dir, filename)
    source_info = get_source_info(git_path)
    self.save_json(source_info, filename)

    self.logger.info(f"{type(self).__name__} source info is saved in {save_path}.")

save_checkpoint 

save_checkpoint(trainer: BaseTrainer, filename: str, password: Optional[str] = None) -> None

Save checkpoint in the "checkpoint" directory.

Parameters:

  • trainer (Trainer) –

    An instance containing information to be saved in the checkpoint.

  • filename (str) –

    filename

  • password (Optional[str], default: None ) –

    Password for checkpoint file. Defaults to None.

Source code in SaigeToolkit/log/log_handler.py 
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
@skip_if_logdir_is_none
def save_checkpoint(
    self, trainer: BaseTrainer, filename: str, password: Optional[str] = None
) -> None:
    """Save checkpoint in the "checkpoint" directory.

    Args:
        trainer (Trainer): An instance containing information to be saved in the checkpoint.
        filename (str): filename
        password (Optional[str], optional): Password for checkpoint file. Defaults to None.
    """
    save_dir = os.path.join(self.logdir, "checkpoint")
    save_path = os.path.join(save_dir, filename)
    os.makedirs(os.path.dirname(save_path), exist_ok=True)

    trainer.save_checkpoint(save_path, password=password)

configure_logger 

configure_logger(name: Optional[str] = 'SaigeResearch', level: str = 'INFO', formatter: str = f'%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s', path: Optional[str] = None, stdout: bool = False)
Source code in SaigeToolkit/log/logger.py 
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
def configure_logger(
    name: Optional[str] = "SaigeResearch",
    level: str = "INFO",
    formatter: str = f"%(asctime)s %(levelname)s [%(filename)s:%(lineno)d] %(message)s",
    path: Optional[str] = None,
    stdout: bool = False,
):
    logger = logging.getLogger(name)

    logger.setLevel(level)
    logger.propagate = False
    logger.handlers = []

    if path is not None:
        handler = logging.FileHandler(path, mode="a")
        handler.setFormatter(logging.Formatter(formatter))
        logger.addHandler(handler)

    if stdout:
        handler = logging.StreamHandler(sys.stdout)
        handler.setFormatter(logging.Formatter(formatter))
        logger.addHandler(handler)

    return logger

build_dashboard 

build_dashboard(_target_: str = 'Tensorboard', **kwargs) -> Dashboard
Source code in SaigeToolkit/log/dashboard.py 
51
52
def build_dashboard(_target_: str = "Tensorboard", **kwargs) -> Dashboard:
    return Dashboard.registry[_target_](**kwargs)