scheduler

learning.scheduler

torch learning rate schdulers and Few custom schedulers

logger module-attribute

logger = getLogger('SaigeResearch')

_types module-attribute

_types = {__name__: _t7KEfor _type in _types}

model module-attribute

model = FooModule()

optimizer module-attribute

optimizer = build_optimizer(parameters(), **{'_target_': 'SGD', 'lr': 0.0, 'momentum': 0.9, 'weight_decay': 0.0001})

scheduler module-attribute

scheduler = build_scheduler(optimizer, **{'_target_': 'CosineAnnealingWarmUpRestarts', 'T_0': 50000, 'T_mult': 1, 'eta_max': 0.005, 'T_up': 500, 'gamma': 0.5})

FixedLR

FixedLR(optimizer: Optimizer, last_epoch: int = -1)

Bases: _LRScheduler

constant learning rate

initializing FixedLR

Parameters:

• optimizer (Optimizer) –

  optimizer

• last_epoch (int, default: -1 ) –

  index of last epoch. Defaults to -1.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer: Optimizer, last_epoch: int = -1) -> None:
    """initializing FixedLR

    Args:
        optimizer (Optimizer): optimizer
        last_epoch (int, optional): index of last epoch. Defaults to -1.
    """
    super(FixedLR, self).__init__(optimizer, last_epoch)

get_lr

get_lr()

Source code in SaigeToolkit/learning/scheduler.py

def get_lr(self):
    return [base_lr for base_lr in self.base_lrs]

PolynomialLR

PolynomialLR(optimizer: Optimizer, max_iter: int, decay_iter: int = 1, gamma: float = 0.9, last_epoch: int = -1)

Bases: _LRScheduler

polynomial learning rate scheduler

Attributes:

• decay_iter (int) –

  decaying iteration number

• max_iter (int) –

  max iteration number for decaying

• gamma (float) –

  learning rate decay rate (exponents)

initializing PolynomialLR

Parameters:

• optimizer (Optimizer) –

  optimizer

• max_iter (int) –

  max iteration number for decaying

• decay_iter (int, default: 1 ) –

  decaying iteration number. Defaults to 1.

• gamma (float, default: 0.9 ) –

  learning rate decay rate (exponents). Defaults to 0.9.

• last_epoch (int, default: -1 ) –

  index of last epoch. Defaults to -1.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    max_iter: int,
    decay_iter: int = 1,
    gamma: float = 0.9,
    last_epoch: int = -1,
) -> None:
    """initializing PolynomialLR

    Args:
        optimizer (Optimizer): optimizer
        max_iter (int): max iteration number for decaying
        decay_iter (int, optional): decaying iteration number. Defaults to 1.
        gamma (float, optional): learning rate decay rate (exponents). Defaults to 0.9.
        last_epoch (int, optional): index of last epoch. Defaults to -1.
    """
    self.decay_iter = decay_iter
    self.max_iter = max_iter
    self.gamma = gamma
    super(PolynomialLR, self).__init__(optimizer, last_epoch)

decay_iter instance-attribute

decay_iter = decay_iter

max_iter instance-attribute

max_iter = max_iter

gamma instance-attribute

gamma = gamma

get_lr

get_lr()

Source code in SaigeToolkit/learning/scheduler.py

def get_lr(self):
    if self.last_epoch % self.decay_iter or self.last_epoch % self.max_iter:
        return [base_lr for base_lr in self.base_lrs]
    else:
        factor = (1 - self.last_epoch / float(self.max_iter)) ** self.gamma
        return [base_lr * factor for base_lr in self.base_lrs]

WarmUpLR

WarmUpLR(optimizer: Optimizer, scheduler: dict, mode: str = 'linear', warmup_iters: int = 100, gamma: float = 0.2)

Bases: _LRScheduler

Wrapper for learning rate scheduler with warm-up stage

Attributes:

• mode (str) –

  base schduler mode after warm-up stage

• scheduler (_LRScheduler) –

  schduler for cold_lrs (warm-up stage)

• warmup_iters (int) –

  number of iterations for warm-up stage

• gamma (float) –

  learning rate decay rate

Parameters:

• scheduler (dict) –

  schduler dict for cold_lrs (warm-up stage)

• mode (str, default: 'linear' ) –

  base schduler mode after warm-up stage. Defaults to "linear".

• warmup_iters (int, default: 100 ) –

  number of iterations for warm-up stage. Defaults to 100.

• gamma (float, default: 0.2 ) –

  learning rate decay ratev. Defaults to 0.2.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    scheduler: dict,
    mode: str = "linear",
    warmup_iters: int = 100,
    gamma: float = 0.2,
) -> None:
    """
    Args:
        scheduler (dict): schduler dict for cold_lrs (warm-up stage)
        mode (str, optional): base schduler mode after warm-up stage. Defaults to "linear".
        warmup_iters (int, optional): number of iterations for warm-up stage. Defaults to 100.
        gamma (float, optional): learning rate decay ratev. Defaults to 0.2.
    """
    self._warmup_strategy = mode
    if self._warmup_strategy == "cos":
        self._warmup_func = self._warmup_cos
    elif self._warmup_strategy == "linear":
        self._warmup_func = self._warmup_linear
    elif self._warmup_strategy == "const":
        self._warmup_func = self._warmup_const
    else:
        raise NotImplementedError(f"Warmup type {self._warmup_strategy} not implemented.")

    self.gamma = gamma

    self._scheduler = build_scheduler(optimizer=optimizer, **scheduler)
    self._init_lr = self._scheduler.optimizer.param_groups[0]["lr"] * self.gamma
    self._warmup_iters = warmup_iters
    self._step_count = 0
    self._format_param()

_warmup_strategy instance-attribute

_warmup_strategy = mode

_warmup_func instance-attribute

_warmup_func = _warmup_cos

gamma instance-attribute

gamma = gamma

_scheduler instance-attribute

_scheduler = build_scheduler(optimizer=optimizer, **scheduler)

_init_lr instance-attribute

_init_lr = param_groups[0]['lr'] * gamma

_warmup_iters instance-attribute

_warmup_iters = warmup_iters

_step_count instance-attribute

_step_count = 0

_format_param

_format_param()

Source code in SaigeToolkit/learning/scheduler.py

def _format_param(self):
    # learning rate of each param group will increase
    # from the min_lr to initial_lr
    for group in self._scheduler.optimizer.param_groups:
        group["warmup_max_lr"] = group["lr"]
        group["warmup_initial_lr"] = min(self._init_lr, group["lr"])

__getattr__

__getattr__(name)

Source code in SaigeToolkit/learning/scheduler.py

def __getattr__(self, name):
    return getattr(self._scheduler, name)

state_dict

state_dict()

Returns the state of the scheduler as a :class:dict. It contains an entry for every variable in self.dict which is not the optimizer.

Source code in SaigeToolkit/learning/scheduler.py

def state_dict(self):
    """Returns the state of the scheduler as a :class:`dict`.
    It contains an entry for every variable in self.__dict__ which
    is not the optimizer.
    """
    wrapper_state_dict = {
        key: value for key, value in self.__dict__.items() if key not in ["optimizer", "_scheduler"]
    }
    wrapped_state_dict = {
        key: value for key, value in self._scheduler.__dict__.items() if key != "optimizer"
    }
    return {"wrapped": wrapped_state_dict, "wrapper": wrapper_state_dict}

load_state_dict

load_state_dict(state_dict: dict)

Loads the schedulers state.

Parameters:

• state_dict (dict) –

  scheduler state. Should be an object returned from a call to :meth:state_dict.

Source code in SaigeToolkit/learning/scheduler.py

def load_state_dict(self, state_dict: dict):
    """Loads the schedulers state.

    Args:
        state_dict (dict): scheduler state. Should be an object returned
            from a call to :meth:`state_dict`.
    """
    self.__dict__.update(state_dict["wrapper"])
    self._scheduler.__dict__.update(state_dict["wrapped"])

_warmup_cos

_warmup_cos(start, end, pct)

Source code in SaigeToolkit/learning/scheduler.py

def _warmup_cos(self, start, end, pct):
    cos_out = math.cos(math.pi * pct) + 1
    return end + (start - end) / 2.0 * cos_out

_warmup_const

_warmup_const(start, end, pct)

Source code in SaigeToolkit/learning/scheduler.py

def _warmup_const(self, start, end, pct):
    return start if pct < 0.9999 else end

_warmup_linear

_warmup_linear(start, end, pct)

Source code in SaigeToolkit/learning/scheduler.py

def _warmup_linear(self, start, end, pct):
    return (end - start) * pct + start

step

step(*args)

Source code in SaigeToolkit/learning/scheduler.py

def step(self, *args):
    if self._step_count <= self._warmup_iters:
        values = self.get_lr()
        for param_group, lr in zip(self._scheduler.optimizer.param_groups, values):
            param_group["lr"] = lr
        self._step_count += 1
    else:
        self._scheduler.step(*args)

    self._last_lr = [param_group["lr"] for param_group in self._scheduler.optimizer.param_groups]

get_lr

get_lr()

Source code in SaigeToolkit/learning/scheduler.py

def get_lr(self):
    lrs = []
    # warm up learning rate
    if self._step_count <= self._warmup_iters:
        for group in self._scheduler.optimizer.param_groups:
            computed_lr = self._warmup_func(
                group["warmup_initial_lr"],
                group["warmup_max_lr"],
                self._step_count / self._warmup_iters,
            )
            lrs.append(computed_lr)
    else:
        lrs = self._scheduler.get_lr()
    return lrs

CosineAnnealingWarmUpRestarts

CosineAnnealingWarmUpRestarts(optimizer: Optimizer, T_0: int, T_mult: int = 1, eta_max: float = 0.1, T_up: int = 0, gamma: float = 1.0, last_epoch: int = -1)

Bases: _LRScheduler

Cosine Anneeling scheduler with Warmup restarts (SGDR).

Modified from https://github.com/pytorch/pytorch/blob/v1.1.0/torch/optim/lr_scheduler.py#L655, implementing initial warmup stage.

Usage:

optimizer:
    _target_: sgd
    lr: 0.0  # optimizer lr should be zero or very small value!!!
    momentum: 0.9
    weight_decay: 0.0001
scheduler:
    _target_: CosineAnnealingWarmUpRestarts
    T_0: 50000
    T_mult: 1
    eta_max: 0.005
    T_up: 500
    gamma: 0.5

Parameters:

• optimizer (Optimizer) –

  Pytorch optimizer instance.

• T_0 (int) –

  Initial annealing cycle length.

• T_mult (int, default: 1 ) –

  Cycle multiplication scale after the first cycle. Defaults to 1.

• eta_max (float, default: 0.1 ) –

  Max learning rate. Defaults to 0.1.

• T_up (int, default: 0 ) –

  Warmup step size. Defaults to 0.

• gamma (float, default: 1.0 ) –

  eta_max multiplication scale after the first cycle. Defaults to 1..

• last_epoch (int, default: -1 ) –

  Last epoch of the scheduler. Defaults to -1.

Raises:

• ValueError –

  Expected positive integer T_0

• ValueError –

  Expected integer T_mult >= 1

• ValueError –

  Expected positive integer T_up

Source code in SaigeToolkit/learning/scheduler.py

def __init__(
    self,
    optimizer: Optimizer,
    T_0: int,
    T_mult: int = 1,
    eta_max: float = 0.1,
    T_up: int = 0,
    gamma: float = 1.0,
    last_epoch: int = -1,
):
    """
    Args:
        optimizer (Optimizer): Pytorch optimizer instance.
        T_0 (int): Initial annealing cycle length.
        T_mult (int, optional): Cycle multiplication scale after the first cycle. Defaults to 1.
        eta_max (float, optional): Max learning rate. Defaults to 0.1.
        T_up (int, optional): Warmup step size. Defaults to 0.
        gamma (float, optional): eta_max multiplication scale after the first cycle. Defaults to 1..
        last_epoch (int, optional): Last epoch of the scheduler. Defaults to -1.

    Raises:
        ValueError: Expected positive integer T_0
        ValueError: Expected integer T_mult >= 1
        ValueError: Expected positive integer T_up
    """
    if T_0 <= 0 or not isinstance(T_0, int):
        raise ValueError(f"Expected positive integer T_0, but got {T_0}")
    if T_mult < 1 or not isinstance(T_mult, int):
        raise ValueError(f"Expected integer T_mult >= 1, but got {T_mult}")
    if T_up < 0 or not isinstance(T_up, int):
        raise ValueError(f"Expected positive integer T_up, but got {T_up}")
    self.T_0 = T_0
    self.T_mult = T_mult
    self.base_eta_max = eta_max
    self.eta_max = eta_max
    self.T_up = T_up
    self.T_i = T_0
    self.gamma = gamma
    self.cycle = 0
    self.T_cur = last_epoch
    super(CosineAnnealingWarmUpRestarts, self).__init__(optimizer, last_epoch)

T_0 instance-attribute

T_0 = T_0

T_mult instance-attribute

T_mult = T_mult

base_eta_max instance-attribute

base_eta_max = eta_max

eta_max instance-attribute

eta_max = eta_max

T_up instance-attribute

T_up = T_up

T_i instance-attribute

T_i = T_0

gamma instance-attribute

gamma = gamma

cycle instance-attribute

cycle = 0

T_cur instance-attribute

T_cur = last_epoch

get_lr

get_lr()

Source code in SaigeToolkit/learning/scheduler.py

def get_lr(self):
    if self.T_cur == -1:
        return self.base_lrs
    elif self.T_cur < self.T_up:
        return [
            (self.eta_max - base_lr) * self.T_cur / self.T_up + base_lr for base_lr in self.base_lrs
        ]
    else:
        return [
            base_lr
            + (self.eta_max - base_lr)
            * (1 + math.cos(math.pi * (self.T_cur - self.T_up) / (self.T_i - self.T_up)))
            / 2
            for base_lr in self.base_lrs
        ]

step

step(epoch=None)

Source code in SaigeToolkit/learning/scheduler.py

def step(self, epoch=None):
    if epoch is None:
        epoch = self.last_epoch + 1
        self.T_cur = self.T_cur + 1
        if self.T_cur >= self.T_i:
            self.cycle += 1
            self.T_cur = self.T_cur - self.T_i
            self.T_i = (self.T_i - self.T_up) * self.T_mult + self.T_up
    else:
        if epoch >= self.T_0:
            if self.T_mult == 1:
                self.T_cur = epoch % self.T_0
                self.cycle = epoch // self.T_0
            else:
                n = int(math.log((epoch / self.T_0 * (self.T_mult - 1) + 1), self.T_mult))
                self.cycle = n
                self.T_cur = epoch - self.T_0 * (self.T_mult**n - 1) / (self.T_mult - 1)
                self.T_i = self.T_0 * self.T_mult ** (n)
        else:
            self.T_i = self.T_0
            self.T_cur = epoch

    self.eta_max = self.base_eta_max * (self.gamma**self.cycle)
    self.last_epoch = math.floor(epoch)

    for param_group, lr in zip(self.optimizer.param_groups, self.get_lr()):
        param_group["lr"] = lr

    self._last_lr = [param_group["lr"] for param_group in self.optimizer.param_groups]

ProportionalMultiStepLR

ProportionalMultiStepLR(optimizer, iteration: int, milestones: Sequence[float], **kwargs)

Bases: MultiStepLR

전체 iteration에 대한 비율로 milestone을 설정하는 MultiStepLR 스케줄러

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer, iteration: int, milestones: Sequence[float], **kwargs) -> None:
    milestones_ = [int(iteration * proportion) for proportion in sorted(milestones)]
    super().__init__(optimizer=optimizer, milestones=milestones_, **kwargs)

SaigeVision1_5080_LR

SaigeVision1_5080_LR(optimizer, iteration: int, **kwargs)

Bases: ProportionalMultiStepLR

SaigeVision1 제품에 적용된 스케줄러. 전체 iteration의 50%, 80% 지점에서 learning rate을 0.5배씩 줄입니다.

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self, optimizer, iteration: int, **kwargs) -> None:
    super().__init__(optimizer, iteration=iteration, milestones=[0.5, 0.8], gamma=0.5, **kwargs)

FooModule

FooModule()

Bases: Module

Source code in SaigeToolkit/learning/scheduler.py

def __init__(self):
    super(FooModule, self).__init__()
    self.conv = torch.nn.Conv2d(3, 3, 3)

conv instance-attribute

conv = Conv2d(3, 3, 3)

forward

forward(x)

Source code in SaigeToolkit/learning/scheduler.py

def forward(self, x):
    return self.conv(x)

build_scheduler

build_scheduler(optimizer: Optimizer, _target_: str = 'FixedLR', **params) -> _LRScheduler

builds leraning rate scheduler

Parameters:

• optimizer (Optimizer) –

  optimizer object

• _target_ (str, default: 'FixedLR' ) –

  learning rate scheduler configuration dict. Defaults to "FixedLR".

Raises:

• NotImplementedError –

  description

Returns:

• _LRScheduler ( _LRScheduler ) –

  learning rate scheduler object

Source code in SaigeToolkit/learning/scheduler.py

def build_scheduler(optimizer: Optimizer, _target_: str = "FixedLR", **params) -> _LRScheduler:
    """builds leraning rate scheduler

    Args:
        optimizer (Optimizer): optimizer object
        _target_ (str, optional): learning rate scheduler configuration dict. Defaults to "FixedLR".

    Raises:
        NotImplementedError: _description_

    Returns:
        _LRScheduler: learning rate scheduler object
    """
    if _target_ not in _types:
        raise NotImplementedError(f"SCHEDULER {_target_} not implemented")
    logger.info(f"[{'SCHEDULER'.center(9)}] {_target_} [params] {params}")
    return _types[_target_](optimizer, **params)

visualize_scheduler

visualize_scheduler(optimizer, scheduler, epochs)

Source code in SaigeToolkit/learning/scheduler.py

def visualize_scheduler(optimizer, scheduler, epochs):
    lrs = []
    for _ in range(epochs):
        optimizer.step()
        lrs.append(scheduler.get_lr())
        scheduler.step()

    plt.plot(lrs)
    plt.savefig("lr_scheduler_test.png", dpi=300)