scheduler

learning.scheduler

torch learning rate schdulers and Few custom schedulers

logger module-attribute

logger = getLogger('SaigeResearch')

_types module-attribute

_types = {__name__: _V2Ybfor _type in _types}

model module-attribute

model = FooModule()

optimizer module-attribute

optimizer = build_optimizer(parameters(), **{'_target_': 'SGD', 'lr': 0.0, 'momentum': 0.9, 'weight_decay': 0.0001})

scheduler module-attribute

scheduler = build_scheduler(optimizer, **{'_target_': 'CosineAnnealingWarmUpRestarts', 'T_0': 50000, 'T_mult': 1, 'eta_max': 0.005, 'T_up': 500, 'gamma': 0.5})

FixedLR

FixedLR(optimizer: Optimizer, last_epoch: int = -1)

Bases: _LRScheduler

constant learning rate

initializing FixedLR

Parameters:

• optimizer (Optimizer) –

  optimizer

• last_epoch (int, default: -1 ) –

  index of last epoch. Defaults to -1.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer: Optimizer, last_epoch: int = -1) -> None:
    """initializing FixedLR

    Args:
        optimizer (Optimizer): optimizer
        last_epoch (int, optional): index of last epoch. Defaults to -1.
    """
    super(FixedLR, self).__init__(optimizer, last_epoch)

PolynomialLR

PolynomialLR(optimizer: Optimizer, max_iter: int, decay_iter: int = 1, gamma: float = 0.9, last_epoch: int = -1)

Bases: _LRScheduler

polynomial learning rate scheduler

Attributes:

• decay_iter (int) –

  decaying iteration number

• max_iter (int) –

  max iteration number for decaying

• gamma (float) –

  learning rate decay rate (exponents)

initializing PolynomialLR

Parameters:

• optimizer (Optimizer) –

  optimizer

• max_iter (int) –

  max iteration number for decaying

• decay_iter (int, default: 1 ) –

  decaying iteration number. Defaults to 1.

• gamma (float, default: 0.9 ) –

  learning rate decay rate (exponents). Defaults to 0.9.

• last_epoch (int, default: -1 ) –

  index of last epoch. Defaults to -1.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    max_iter: int,
    decay_iter: int = 1,
    gamma: float = 0.9,
    last_epoch: int = -1,
) -> None:
    """initializing PolynomialLR

    Args:
        optimizer (Optimizer): optimizer
        max_iter (int): max iteration number for decaying
        decay_iter (int, optional): decaying iteration number. Defaults to 1.
        gamma (float, optional): learning rate decay rate (exponents). Defaults to 0.9.
        last_epoch (int, optional): index of last epoch. Defaults to -1.
    """
    self.decay_iter = decay_iter
    self.max_iter = max_iter
    self.gamma = gamma
    super(PolynomialLR, self).__init__(optimizer, last_epoch)

WarmUpLR

WarmUpLR(optimizer: Optimizer, scheduler: dict, mode: str = 'linear', warmup_iters: int = 100, gamma: float = 0.2)

Bases: _LRScheduler

Wrapper for learning rate scheduler with warm-up stage

Attributes:

• mode (str) –

  base schduler mode after warm-up stage

• scheduler (_LRScheduler) –

  schduler for cold_lrs (warm-up stage)

• warmup_iters (int) –

  number of iterations for warm-up stage

• gamma (float) –

  learning rate decay rate

Parameters:

• scheduler (dict) –

  schduler dict for cold_lrs (warm-up stage)

• mode (str, default: 'linear' ) –

  base schduler mode after warm-up stage. Defaults to "linear".

• warmup_iters (int, default: 100 ) –

  number of iterations for warm-up stage. Defaults to 100.

• gamma (float, default: 0.2 ) –

  learning rate decay ratev. Defaults to 0.2.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    scheduler: dict,
    mode: str = "linear",
    warmup_iters: int = 100,
    gamma: float = 0.2,
) -> None:
    """
    Args:
        scheduler (dict): schduler dict for cold_lrs (warm-up stage)
        mode (str, optional): base schduler mode after warm-up stage. Defaults to "linear".
        warmup_iters (int, optional): number of iterations for warm-up stage. Defaults to 100.
        gamma (float, optional): learning rate decay ratev. Defaults to 0.2.
    """
    self._warmup_strategy = mode
    if self._warmup_strategy == "cos":
        self._warmup_func = self._warmup_cos
    elif self._warmup_strategy == "linear":
        self._warmup_func = self._warmup_linear
    elif self._warmup_strategy == "const":
        self._warmup_func = self._warmup_const
    else:
        raise NotImplementedError(f"Warmup type {self._warmup_strategy} not implemented.")

    self.gamma = gamma

    self._scheduler = build_scheduler(optimizer=optimizer, **scheduler)
    self._init_lr = self._scheduler.optimizer.param_groups[0]["lr"] * self.gamma
    self._warmup_iters = warmup_iters
    self._step_count = 0
    self._format_param()

state_dict

state_dict()

Returns the state of the scheduler as a :class:dict. It contains an entry for every variable in self.dict which is not the optimizer.

Source code in SaigeToolkit/learning/scheduler.py

def state_dict(self):
    """Returns the state of the scheduler as a :class:`dict`.
    It contains an entry for every variable in self.__dict__ which
    is not the optimizer.
    """
    wrapper_state_dict = {
        key: value for key, value in self.__dict__.items() if key not in ["optimizer", "_scheduler"]
    }
    wrapped_state_dict = {
        key: value for key, value in self._scheduler.__dict__.items() if key != "optimizer"
    }
    return {"wrapped": wrapped_state_dict, "wrapper": wrapper_state_dict}

load_state_dict

load_state_dict(state_dict: dict)

Loads the schedulers state.

Parameters:

• state_dict (dict) –

  scheduler state. Should be an object returned from a call to :meth:state_dict.

Source code in SaigeToolkit/learning/scheduler.py

def load_state_dict(self, state_dict: dict):
    """Loads the schedulers state.

    Args:
        state_dict (dict): scheduler state. Should be an object returned
            from a call to :meth:`state_dict`.
    """
    self.__dict__.update(state_dict["wrapper"])
    self._scheduler.__dict__.update(state_dict["wrapped"])

CosineAnnealingWarmUpRestarts

CosineAnnealingWarmUpRestarts(optimizer: Optimizer, T_0: int, T_mult: int = 1, eta_max: float = 0.1, T_up: int = 0, gamma: float = 1.0, last_epoch: int = -1)

Bases: _LRScheduler

Cosine Anneeling scheduler with Warmup restarts (SGDR).

Modified from https://github.com/pytorch/pytorch/blob/v1.1.0/torch/optim/lr_scheduler.py#L655, implementing initial warmup stage.

Usage:

optimizer: ... target: sgd ... lr: 0.0 # optimizer lr should be zero or very small value!!! ... momentum: 0.9 ... weight_decay: 0.0001 ... scheduler: ... target: CosineAnnealingWarmUpRestarts ... T_0: 50000 ... T_mult: 1 ... eta_max: 0.005 ... T_up: 500 ... gamma: 0.5

Parameters:

• optimizer (Optimizer) –

  Pytorch optimizer instance.

• T_0 (int) –

  Initial annealing cycle length.

• T_mult (int, default: 1 ) –

  Cycle multiplication scale after the first cycle. Defaults to 1.

• eta_max (float, default: 0.1 ) –

  Max learning rate. Defaults to 0.1.

• T_up (int, default: 0 ) –

  Warmup step size. Defaults to 0.

• gamma (float, default: 1.0 ) –

  eta_max multiplication scale after the first cycle. Defaults to 1..

• last_epoch (int, default: -1 ) –

  Last epoch of the scheduler. Defaults to -1.

Raises:

• ValueError –

  Expected positive integer T_0

• ValueError –

  Expected integer T_mult >= 1

• ValueError –

  Expected positive integer T_up

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    T_0: int,
    T_mult: int = 1,
    eta_max: float = 0.1,
    T_up: int = 0,
    gamma: float = 1.0,
    last_epoch: int = -1,
):
    """
    Args:
        optimizer (Optimizer): Pytorch optimizer instance.
        T_0 (int): Initial annealing cycle length.
        T_mult (int, optional): Cycle multiplication scale after the first cycle. Defaults to 1.
        eta_max (float, optional): Max learning rate. Defaults to 0.1.
        T_up (int, optional): Warmup step size. Defaults to 0.
        gamma (float, optional): eta_max multiplication scale after the first cycle. Defaults to 1..
        last_epoch (int, optional): Last epoch of the scheduler. Defaults to -1.

    Raises:
        ValueError: Expected positive integer T_0
        ValueError: Expected integer T_mult >= 1
        ValueError: Expected positive integer T_up
    """
    if T_0 <= 0 or not isinstance(T_0, int):
        raise ValueError(f"Expected positive integer T_0, but got {T_0}")
    if T_mult < 1 or not isinstance(T_mult, int):
        raise ValueError(f"Expected integer T_mult >= 1, but got {T_mult}")
    if T_up < 0 or not isinstance(T_up, int):
        raise ValueError(f"Expected positive integer T_up, but got {T_up}")
    self.T_0 = T_0
    self.T_mult = T_mult
    self.base_eta_max = eta_max
    self.eta_max = eta_max
    self.T_up = T_up
    self.T_i = T_0
    self.gamma = gamma
    self.cycle = 0
    self.T_cur = last_epoch
    super(CosineAnnealingWarmUpRestarts, self).__init__(optimizer, last_epoch)

ProportionalMultiStepLR

ProportionalMultiStepLR(optimizer, iteration: int, milestones: Sequence[float], **kwargs)

Bases: MultiStepLR

전체 iteration에 대한 비율로 milestone을 설정하는 MultiStepLR 스케줄러

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer, iteration: int, milestones: Sequence[float], **kwargs) -> None:
    milestones_ = [int(iteration * proportion) for proportion in sorted(milestones)]
    super().__init__(optimizer=optimizer, milestones=milestones_, **kwargs)

SaigeVision1_5080_LR

SaigeVision1_5080_LR(optimizer, iteration: int, **kwargs)

Bases: ProportionalMultiStepLR

SaigeVision1 제품에 적용된 스케줄러. 전체 iteration의 50%, 80% 지점에서 learning rate을 0.5배씩 줄입니다.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer, iteration: int, **kwargs) -> None:
    super().__init__(optimizer, iteration=iteration, milestones=[0.5, 0.8], gamma=0.5, **kwargs)

FooModule

FooModule()

Bases: Module

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self):
    super(FooModule, self).__init__()
    self.conv = torch.nn.Conv2d(3, 3, 3)

build_scheduler

build_scheduler(optimizer: Optimizer, _target_: str = 'FixedLR', **params) -> _LRScheduler

builds leraning rate scheduler

Parameters:

• optimizer (Optimizer) –

  optimizer object

• _target_ (str, default: 'FixedLR' ) –

  learning rate scheduler configuration dict. Defaults to "FixedLR".

Raises:

• NotImplementedError –

  description

Returns:

• _LRScheduler ( _LRScheduler ) –

  learning rate scheduler object

Source code in SaigeToolkit/learning/scheduler.py

def build_scheduler(optimizer: Optimizer, _target_: str = "FixedLR", **params) -> _LRScheduler:
    """builds leraning rate scheduler

    Args:
        optimizer (Optimizer): optimizer object
        _target_ (str, optional): learning rate scheduler configuration dict. Defaults to "FixedLR".

    Raises:
        NotImplementedError: _description_

    Returns:
        _LRScheduler: learning rate scheduler object
    """
    if _target_ not in _types:
        raise NotImplementedError(f"SCHEDULER {_target_} not implemented")
    logger.info(f"[{'SCHEDULER'.center(9)}] {_target_} [params] {params}")
    return _types[_target_](optimizer, **params)

visualize_scheduler

visualize_scheduler(optimizer, scheduler, epochs)

Source code in SaigeToolkit/learning/scheduler.py

def visualize_scheduler(optimizer, scheduler, epochs):
    lrs = []
    for _ in range(epochs):
        optimizer.step()
        lrs.append(scheduler.get_lr())
        scheduler.step()

    plt.plot(lrs)
    plt.savefig("lr_scheduler_test.png", dpi=300)