Skip to content

benchmark

model.benchmark

ModelSpecInspector

measure_model_inference_spec classmethod 

measure_model_inference_spec(model: Module, batch: int = 1, image_size: int = 256, image_channels: int = 3, warmup: int = 3, device: device = torch.device('cuda'), num_iteration: int = 100)

모델 인퍼런스 시 속도와 메모리 등을 측정합니다.

Source code in SaigeToolkit/model/benchmark.py 
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
@classmethod
def measure_model_inference_spec(
    cls,
    model: torch.nn.Module,
    batch: int = 1,
    image_size: int = 256,
    image_channels: int = 3,
    warmup: int = 3,
    device: torch.device = torch.device("cuda"),
    num_iteration: int = 100,
):
    """모델 인퍼런스 시 속도와 메모리 등을 측정합니다."""
    model.to(device)
    model.eval()

    dummy_input = torch.zeros(batch, image_channels, image_size, image_size)
    dummy_input = dummy_input.to(device)

    timer = Timer(device)

    info = {
        "system": {
            "device": get_torch_device_name(device=device),
            **get_system_info(),
            **get_torch_info(),
        },
        "options": {
            "batch": batch,
            "image_size": image_size,
            "image_channels": image_channels,
            "warmup": warmup,
            "num_iteration": num_iteration,
        },
    }

    try:
        with torch.inference_mode():
            for _ in range(warmup):
                _ = model(dummy_input)

            with timer:
                for _ in range(num_iteration):
                    _ = model(dummy_input)

        total_time = timer.accumulated_time

    except RuntimeError as error:
        if "CUDA out of memory" in str(error):
            total_time = 0
        else:
            raise error

    time_per_batch = total_time / num_iteration
    time_per_image = time_per_batch / batch

    info["results"] = {
        "total_time(ms)": total_time,
        "ms/batch": time_per_batch,
        "ms/image": time_per_image,
        **get_memory_stats(),
    }

    return info

measure_and_save_model_inference_spec_from_config classmethod 

measure_and_save_model_inference_spec_from_config(model: Union[str, Dict], json_path: str, model_builder: Callable = build_backbone, **measure_kwargs)

모델 config 또는 config 파일의 경로를 받아서 모델 인퍼런스 스펙 측정 후 json_path에 결과를 저장합니다.

Source code in SaigeToolkit/model/benchmark.py 
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
@classmethod
def measure_and_save_model_inference_spec_from_config(
    cls,
    model: Union[str, Dict],
    json_path: str,
    model_builder: Callable = build_backbone,
    **measure_kwargs,
):
    """모델 config 또는 config 파일의 경로를 받아서 모델 인퍼런스 스펙 측정 후 json_path에 결과를 저장합니다."""
    if isinstance(model, str) and model.lower().endswith((".yml", ".yaml")):
        model = _load_yml(model)
    model_instance = model_builder(**model)
    results = cls.measure_model_inference_spec(model=model_instance, **measure_kwargs)
    results["options"]["model"] = model
    os.makedirs(os.path.dirname(json_path), exist_ok=True)
    with open(json_path, "w") as f:
        json.dump(results, f, sort_keys=False, indent=4)

benchmark_model_inference_spec classmethod 

benchmark_model_inference_spec(logdir, add_time_to_logdir: bool = True, image_sizes=[256, 512, 1024, 2048], **measure_kwargs)

여러 이미지 사이즈에 인퍼런스 스펙을 별도 프로세스로 측정하고 logdir 하위에 json들로 저장합니다.

Source code in SaigeToolkit/model/benchmark.py 
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
@classmethod
def benchmark_model_inference_spec(
    cls,
    logdir,
    add_time_to_logdir: bool = True,
    image_sizes=[256, 512, 1024, 2048],
    **measure_kwargs,
):
    """여러 이미지 사이즈에 인퍼런스 스펙을 별도 프로세스로 측정하고 logdir 하위에 json들로 저장합니다."""
    if add_time_to_logdir:
        logdir = os.path.join(logdir, get_time_string())

    json_paths = []
    for image_size in image_sizes:
        json_path = os.path.join(logdir, f"{image_size}.json")
        json_paths.append(json_path)
        process = multiprocessing.Process(
            target=cls.measure_and_save_model_inference_spec_from_config,
            kwargs=dict(json_path=json_path, image_size=image_size, **measure_kwargs),
        )
        process.start()
        process.join()
    return json_paths

build_backbone 

build_backbone(_target_: str, **params) -> Module
Source code in SaigeToolkit/model/backbone/builder.py 
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
def build_backbone(_target_: str, **params) -> nn.Module:
    if _target_ in native_models:
        builder = native_models[_target_]
    elif _target_.startswith("torchvision.models."):
        builder = getattr(torchvision.models, _target_.split(".")[-1])
    elif _target_ == "timm.create_model":
        import timm

        builder = timm.create_model
    elif _target_ == "torch.hub.load":
        builder = torch.hub.load
    else:
        raise ValueError(f"Unknown backbone target: {_target_}")

    return builder(**params)