Skip to content

fpn

model.neck.fpn

ConvBlock 

ConvBlock(in_channels: int, out_channels: int, kernel_size: int = 1, stride: int = 1, padding: int = 0)

Bases: Module

ConvBlock defined, which consists of Convolution, BatchNorm, ReLU

Attributes:

  • conv (Conv2d) –

    convolution layer for ConvBlock

  • bn (BatchNorm2d) –

    2d batch norm layer for ConvBlock

  • act (ReLU) –

    ReLU activation function for ConvBlock

Author

Sukho Yoon

initializing ConvBlock

Parameters:

  • in_channels (int) –

    'input channel' for conv layer

  • out_channels (int) –

    'output channel' for conv layer, which is also 'input channel' for bn layer

  • kernel_size (int, default: 1 ) –

    kernel size for conv layer. Defaults to 1.

  • stride (int, default: 1 ) –

    stride for conv layer. Defaults to 1.

  • padding (int, default: 0 ) –

    padding for conv layer. Defaults to 0.

Source code in SaigeToolkit/model/neck/fpn.py 
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
def __init__(
    self,
    in_channels: int,
    out_channels: int,
    kernel_size: int = 1,
    stride: int = 1,
    padding: int = 0,
) -> None:
    """initializing ConvBlock

    Args:
        in_channels (int): 'input channel' for conv layer
        out_channels (int): 'output channel' for conv layer, which is also 'input channel' for bn layer
        kernel_size (int, optional): kernel size for conv layer. Defaults to 1.
        stride (int, optional): stride for conv layer. Defaults to 1.
        padding (int, optional): padding for conv layer. Defaults to 0.
    """
    super(ConvBlock, self).__init__()
    self.conv = nn.Conv2d(in_channels, out_channels, kernel_size, stride=stride, padding=padding)
    self.bn = nn.BatchNorm2d(out_channels)
    self.act = nn.ReLU()

forward 

forward(inputs: Tensor) -> Tensor

forward function for nn.Module class

Parameters:

  • inputs (Tensor) –

    input tensor for ConvBlock

Returns:

  • Tensor –

    torch.Tensor: output tensor for ConvBlock

Source code in SaigeToolkit/model/neck/fpn.py 
43
44
45
46
47
48
49
50
51
52
53
54
def forward(self, inputs: torch.Tensor) -> torch.Tensor:
    """forward function for nn.Module class

    Args:
        inputs (torch.Tensor): input tensor for ConvBlock

    Returns:
        torch.Tensor: output tensor for ConvBlock
    """
    x = self.conv(inputs)
    x = self.bn(x)
    return self.act(x)

FpnBlock 

FpnBlock(in_channels: Union[List[int], int, None] = [64, 128, 256, 512], out_channels: Union[List[int], int, None] = None, channel_fpn: int = 64, num_feat: int = 4)

Bases: Module

block for constructing FeaturePyramidNetwork.

Attributes:

  • block_in (ModuleList) –

    Convolution layer, that backbone output feature is directly applied

  • block_out (ModuleList) –

    Convolution layer, that tensors from 'block_in' is applied after 'upsampled and sumed'

initializing FpnBlock

Parameters:

  • in_channels (Union[List[int], int, None], default: [64, 128, 256, 512] ) –

    input channels for block_in. Defaults to [64, 128, 256, 512].

  • out_channels (Union[List[int], int, None], default: None ) –

    output channels for block_out. Defaults to None.

  • channel_fpn (int, default: 64 ) –

    output channels for block_in, also input channels for block_out. Defaults to 64.

  • num_feat (int, default: 4 ) –

    number of layer of block_in/block_out. Defaults to 4.

Source code in SaigeToolkit/model/neck/fpn.py 
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
def __init__(
    self,
    in_channels: Union[List[int], int, None] = [64, 128, 256, 512],
    out_channels: Union[List[int], int, None] = None,
    channel_fpn: int = 64,
    num_feat: int = 4,
) -> None:
    """initializing FpnBlock

    Args:
        in_channels (Union[List[int], int, None], optional):
            input channels for block_in. Defaults to [64, 128, 256, 512].
        out_channels (Union[List[int], int, None], optional):
            output channels for block_out. Defaults to None.
        channel_fpn (int, optional):
            output channels for block_in, also input channels for block_out. Defaults to 64.
        num_feat (int, optional): number of layer of block_in/block_out. Defaults to 4.
    """
    super(FpnBlock, self).__init__()

    if in_channels is None:
        in_channels = [channel_fpn for _ in range(num_feat)]
    elif type(in_channels) == int:
        in_channels = [in_channels for _ in range(num_feat)]
    else:
        assert type(in_channels) == list, f"wrong type of in_channels: {type(in_channels)}"

    if out_channels is None:
        out_channels = [channel_fpn for _ in range(num_feat)]
    elif type(out_channels) == int:
        out_channels = [out_channels for _ in range(num_feat)]
    else:
        assert type(out_channels) == list, f"wrong type of out_channels: {type(in_channels)}"

    self.block_in, self.block_out = [], []

    for c_in, c_out in zip(in_channels, out_channels):
        self.block_in.append(nn.Conv2d(c_in, channel_fpn, kernel_size=1, stride=1, padding=0))
        self.block_out.append(nn.Conv2d(channel_fpn, c_out, kernel_size=3, stride=1, padding=1))

    self.block_in = nn.ModuleList(self.block_in)
    self.block_out = nn.ModuleList(self.block_out)

_upsample_add 

_upsample_add(x: Tensor, y: Tensor) -> Tensor

upsample intput x to size of input y, and sum both inputs

Parameters:

  • x (Tensor) –

    smaller (size) input

  • y (Tensor) –

    larger (size) input

Returns:

  • Tensor –

    torch.Tensor: upsampled and sumed tensor

Source code in SaigeToolkit/model/neck/fpn.py 
110
111
112
113
114
115
116
117
118
119
120
121
def _upsample_add(self, x: torch.Tensor, y: torch.Tensor) -> torch.Tensor:
    """upsample intput `x` to `size of input y`, and sum both inputs

    Args:
        x (torch.Tensor): smaller (size) input
        y (torch.Tensor): larger (size) input

    Returns:
        torch.Tensor: upsampled and sumed tensor
    """
    _, _, H, W = y.size()
    return F.upsample(x, size=(H, W), mode="bilinear") + y

forward 

forward(features: List[Tensor]) -> List[Tensor]

forward function for nn.Module class

Parameters:

  • features (List[Tensor]) –

    list of tensors from backbone network

Returns:

  • List[Tensor] –

    List[torch.Tensor]: list of tensors processed after FpnBlock

Source code in SaigeToolkit/model/neck/fpn.py 
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
def forward(self, features: List[torch.Tensor]) -> List[torch.Tensor]:
    """forward function for nn.Module class

    Args:
        features (List[torch.Tensor]): list of tensors from backbone network

    Returns:
        List[torch.Tensor]: list of tensors processed after FpnBlock
    """
    ls_out = []
    ls_out.insert(0, self.block_in[-1](features.pop(-1)))
    for i in range(len(features) - 1, -1, -1):
        ls_out.insert(0, self._upsample_add(ls_out[0], self.block_in[i](features.pop(-1))))

    for i_o in range(len(ls_out)):
        ls_out[i_o] = self.block_out[i_o](ls_out[i_o])

    return ls_out

FPN 

FPN(in_channels: Optional[List[int]] = None, out_channels: int = 64, channel_fpn: int = 256, num_feat: int = 4, num_block: int = 1)

Bases: Module

"Feature Pyramid Networks for Object Detection".

Attributes:

  • block_fpn (ModuleList) –

    torch module list, consists of FpnBlocks.

initializing Feature Pyramid Networks

Parameters:

  • in_channels (List[int], default: None ) –

    channels from backbone network output tensors. Defaults to None.

  • out_channels (int, default: 64 ) –

    final output channels for FPN. Defaults to 64.

  • channel_fpn (int, default: 256 ) –

    inner channels for FpnBlocks. Defaults to 256.

  • num_feat (int, default: 4 ) –

    number of blocks inside of FpnBlocks. Defaults to 4.

  • num_block (int, default: 1 ) –

    number of FpnBlocks. Defaults to 1.

Source code in SaigeToolkit/model/neck/fpn.py 
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
def __init__(
    self,
    in_channels: Optional[List[int]] = None,
    out_channels: int = 64,
    channel_fpn: int = 256,
    num_feat: int = 4,
    num_block: int = 1,
) -> None:
    """initializing Feature Pyramid Networks

    Args:
        in_channels (List[int], optional): channels from backbone network output tensors. Defaults to None.
        out_channels (int, optional): final output channels for FPN. Defaults to 64.
        channel_fpn (int, optional): inner channels for FpnBlocks. Defaults to 256.
        num_feat (int, optional): number of blocks inside of FpnBlocks. Defaults to 4.
        num_block (int, optional): number of FpnBlocks. Defaults to 1.
    """

    super(FPN, self).__init__()

    if in_channels is None:
        in_channels = [64, 128, 256, 512]

    self.block_fpn = []
    for i_nb in range(num_block):
        c_in = in_channels if i_nb == 0 else None
        c_out = out_channels if i_nb == num_block - 1 else None

        self.block_fpn.append(FpnBlock(c_in, c_out, channel_fpn, num_feat))

    self.block_fpn = nn.ModuleList(self.block_fpn)

forward 

forward(features: List[Tensor]) -> List[Tensor]

forward function for FPN

Parameters:

  • features (List[Tensor]) –

    output tensors from backbone network.

Returns:

  • List[Tensor] –

    List[torch.Tensor]: output tensors from FPN.

Source code in SaigeToolkit/model/neck/fpn.py 
182
183
184
185
186
187
188
189
190
191
192
193
194
def forward(self, features: List[torch.Tensor]) -> List[torch.Tensor]:
    """forward function for FPN

    Args:
        features (List[torch.Tensor]): output tensors from backbone network.

    Returns:
        List[torch.Tensor]: output tensors from FPN.
    """
    for b_fpn in self.block_fpn:
        features = b_fpn(features)

    return features

BiFpnBlock 

BiFpnBlock(channel_fpn: int = 256, num_feat: int = 4, epsilon: float = 1e-06)

Bases: Module

block for constructing BidirectionalFeaturePyramidNetwork.

Attributes:

  • epsilon (float) –

    epsilon value for BiFpnBlock

  • block_td (ModuleList) –

    blocks for upsample_weighted_sum

  • block_out (ModuleList) –

    blocks for downsample_weighted_sum

  • w1 (Parameter) –

    weight parameter for block_td

  • w1_relu (ReLU) –

    ReLU layer for w1 parameter

  • w2 (Parameter) –

    weight parameter for block_out

  • w2_relu (ReLU) –

    ReLU layer for w2 parameter

initializing BiFpnBlock

Parameters:

  • channel_fpn (int, default: 256 ) –

    input/output channels for all blocks. Defaults to 256.

  • num_feat (int, default: 4 ) –

    number of each blocks. Defaults to 4.

  • epsilon (float, default: 1e-06 ) –

    epsilon value for attribute self.epsilon. Defaults to 1e-6.

Source code in SaigeToolkit/model/neck/fpn.py 
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
def __init__(self, channel_fpn: int = 256, num_feat: int = 4, epsilon: float = 1e-6) -> None:
    """initializing BiFpnBlock

    Args:
        channel_fpn (int, optional): input/output channels for all blocks. Defaults to 256.
        num_feat (int, optional): number of each blocks. Defaults to 4.
        epsilon (float, optional): epsilon value for attribute `self.epsilon`. Defaults to 1e-6.
    """
    super(BiFpnBlock, self).__init__()
    self.epsilon = epsilon

    self.block_td = nn.ModuleList([ConvBlock(channel_fpn, channel_fpn) for _ in range(num_feat - 1)])
    self.block_out = nn.ModuleList(
        [ConvBlock(channel_fpn, channel_fpn) for _ in range(num_feat - 1)]
    )

    self.w1 = nn.Parameter(torch.ones((num_feat - 1, 2)))
    self.w1_relu = nn.ReLU()
    self.w2 = nn.Parameter(torch.ones((num_feat - 1, 3)))
    self.w2_relu = nn.ReLU()

forward 

forward(inputs: List[Tensor]) -> List[Tensor]

forward function for nn.Module

Parameters:

  • inputs (List[Tensor]) –

    list of tensors from former network

Returns:

  • List[Tensor] –

    List[torch.Tensor]: output list of tensors from BiFpnBlock

Source code in SaigeToolkit/model/neck/fpn.py 
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
def forward(self, inputs: List[torch.Tensor]) -> List[torch.Tensor]:
    """forward function for nn.Module

    Args:
        inputs (List[torch.Tensor]): list of tensors from former network

    Returns:
        List[torch.Tensor]: output list of tensors from BiFpnBlock
    """

    w1 = self.w1_relu(self.w1)
    w1 /= torch.sum(w1, dim=0) + self.epsilon
    w2 = self.w2_relu(self.w2)
    w2 /= torch.sum(w2, dim=0) + self.epsilon

    p_td = [inputs[-1]]

    for i_td, b_td in enumerate(self.block_td):
        p_td.insert(0, b_td(self._upsample_weighted_sum(inputs[-2 - i_td], p_td[0], w1[i_td])))

    p_out = [p_td.pop(0)]
    for i_out, b_out in enumerate(self.block_out):
        p_out.append(
            b_out(self._downsample_weighted_sum(inputs.pop(1), p_td.pop(0), p_out[-1], w2[i_out]))
        )

    return p_out

_upsample_weighted_sum 

_upsample_weighted_sum(x: Tensor, y: Tensor, w: Parameter) -> Tensor

upsample input x and weighted sum with input y

Parameters:

  • x (Tensor) –

    smaller (size) input

  • y (Tensor) –

    larger (size) input

  • w (Parameter) –

    weight parameter

Returns:

  • Tensor –

    torch.Tensor: 'upsample_weight_sum'ed output

Source code in SaigeToolkit/model/neck/fpn.py 
261
262
263
264
265
266
267
268
269
270
271
272
273
def _upsample_weighted_sum(self, x: torch.Tensor, y: torch.Tensor, w: nn.Parameter) -> torch.Tensor:
    """upsample `input x` and weighted sum with `input y`

    Args:
        x (torch.Tensor): smaller (size) input
        y (torch.Tensor): larger (size) input
        w (nn.Parameter): weight parameter

    Returns:
        torch.Tensor: 'upsample_weight_sum'ed output
    """
    _, _, H, W = x.size()
    return w[0] * x + w[1] * F.interpolate(y, size=(H, W), mode="bilinear", align_corners=True)

_downsample_weighted_sum 

_downsample_weighted_sum(x: Tensor, y: Tensor, z: Tensor, w: Parameter) -> Tensor

downsample input z and weighted sum with input x and input y

Parameters:

  • x (Tensor) –

    smaller (size) input

  • y (Tensor) –

    smaller (size) input

  • z (Tensor) –

    larger (size) input

  • w (Parameter) –

    weight parameter

Returns:

  • Tensor –

    torch.Tensor: 'downsample_weight_sum'ed output

Source code in SaigeToolkit/model/neck/fpn.py 
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
def _downsample_weighted_sum(
    self, x: torch.Tensor, y: torch.Tensor, z: torch.Tensor, w: nn.Parameter
) -> torch.Tensor:
    """downsample `input z` and weighted sum with `input x` and `input y`

    Args:
        x (torch.Tensor): smaller (size) input
        y (torch.Tensor): smaller (size) input
        z (torch.Tensor): larger (size) input
        w (nn.Parameter): weight parameter

    Returns:
        torch.Tensor: 'downsample_weight_sum'ed output
    """
    _, _, H, W = x.size()
    return (
        w[0] * x
        + w[1] * y
        + w[2] * F.interpolate(z, size=(H, W), mode="bilinear", align_corners=True)
    )

BiFPN 

BiFPN(in_channels: Optional[List[int]] = None, out_channels: int = 64, channel_fpn: int = 256, num_feat: int = 4, num_block: int = 2)

Bases: Module

"BidirectionalFeaturePyramidNetwork".

Attribute

block_in (nn.ModuleList): list of conv layers, which transfers input channels into fixed channel_fpn block_ex (nn.ModuleList): list of ConvBlock, residual blocks block_fpn (nn.ModuleList): list of BiFpnBlocks block_rtn (nn.ModuleList): list of ConvBlock, refining BiFpnBlock output

initializing Bidirectional FPN

Args:

in_channels (List[int], optional): channels from backbone network output tensors. Defaults to None.
out_channels (int, optional): final output channels for FPN. Defaults to 64.
channel_fpn (int, optional): inner channels for BiFpnBlocks. Defaults to 256.
num_feat (int, optional): number of blocks inside of BiFpnBlocks. Defaults to 4.
num_block (int, optional): number of FpnBlocks. Defaults to 1.
Source code in SaigeToolkit/model/neck/fpn.py 
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
def __init__(
    self,
    in_channels: Optional[List[int]] = None,
    out_channels: int = 64,
    channel_fpn: int = 256,
    num_feat: int = 4,
    num_block: int = 2,
) -> None:
    """initializing Bidirectional FPN

    Args:

        in_channels (List[int], optional): channels from backbone network output tensors. Defaults to None.
        out_channels (int, optional): final output channels for FPN. Defaults to 64.
        channel_fpn (int, optional): inner channels for BiFpnBlocks. Defaults to 256.
        num_feat (int, optional): number of blocks inside of BiFpnBlocks. Defaults to 4.
        num_block (int, optional): number of FpnBlocks. Defaults to 1.
    """

    super(BiFPN, self).__init__()

    if in_channels is None:
        in_channels = [64, 128, 256, 512]
    num_feat = len(in_channels) if num_feat is None else num_feat

    self.block_in = []
    for c in in_channels:
        self.block_in.append(nn.Conv2d(c, channel_fpn, kernel_size=1, stride=1, padding=0))
    self.block_in = nn.ModuleList(self.block_in)

    ex_channels = [in_channels[-1]] + [channel_fpn for _ in range(num_feat - 1 - len(in_channels))]
    ex_channels = ex_channels[: num_feat - len(in_channels)]

    self.block_ex = []
    for c in ex_channels:
        self.block_ex.append(ConvBlock(c, channel_fpn, kernel_size=3, stride=2, padding=1))
    self.block_ex = nn.ModuleList(self.block_ex)

    self.block_fpn = nn.ModuleList([BiFpnBlock(channel_fpn, num_feat) for _ in range(num_block)])

    self.block_rtn = []
    if out_channels != channel_fpn:
        for _ in range(num_feat):
            self.block_rtn.append(
                nn.Conv2d(channel_fpn, out_channels, kernel_size=1, stride=1, padding=0)
            )
    self.block_rtn = nn.ModuleList(self.block_rtn)

forward 

forward(inputs: List[Tensor]) -> List[Tensor]

forward function for FPN

Parameters:

  • inputs (List[Tensor]) –

    output tensors from backbone network.

Returns:

  • List[Tensor] –

    List[torch.Tensor]: output tensors from BiFPN.

Source code in SaigeToolkit/model/neck/fpn.py 
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
def forward(self, inputs: List[torch.Tensor]) -> List[torch.Tensor]:
    """forward function for FPN

    Args:
        inputs (List[torch.Tensor]): output tensors from backbone network.

    Returns:
        List[torch.Tensor]: output tensors from BiFPN.
    """
    assert len(inputs) == len(self.block_in), "channel number does not fit"

    p_ls = []
    for i_p in range(len(inputs)):
        p_ls.append(self.block_in[i_p](inputs[i_p]))

    if self.block_ex:
        p_ls.append(self.block_ex[0](inputs[-1]))
        for b_ex in self.block_ex[1:]:
            p_ls.append(b_ex(p_ls[-1]))

    for b_fpn in self.block_fpn:
        p_ls = b_fpn(p_ls)

    if self.block_rtn:
        for i_br in range(len(self.block_rtn)):
            p_ls[i_br] = self.block_rtn[i_br](p_ls[i_br])

    return p_ls

FPN_DB 

FPN_DB(in_channels: Optional[List[int]] = None, inner_channels: int = 256, bias: bool = False, *args, **kwargs)

Bases: Module

bias: Whether conv layers have bias or not.

Source code in SaigeToolkit/model/neck/fpn.py 
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
def __init__(
    self,
    in_channels: Optional[List[int]] = None,
    inner_channels: int = 256,
    bias: bool = False,
    *args,
    **kwargs,
):
    """
    bias: Whether conv layers have bias or not.
    """
    super(FPN_DB, self).__init__()

    if in_channels is None:
        in_channels = [64, 128, 256, 512]

    self.up5 = nn.Upsample(scale_factor=2, mode="nearest")
    self.up4 = nn.Upsample(scale_factor=2, mode="nearest")
    self.up3 = nn.Upsample(scale_factor=2, mode="nearest")

    self.in5 = nn.Conv2d(in_channels[-1], inner_channels, 1, bias=bias)
    self.in4 = nn.Conv2d(in_channels[-2], inner_channels, 1, bias=bias)
    self.in3 = nn.Conv2d(in_channels[-3], inner_channels, 1, bias=bias)
    self.in2 = nn.Conv2d(in_channels[-4], inner_channels, 1, bias=bias)

    self.out5 = nn.Conv2d(inner_channels, inner_channels // 4, 3, padding=1, bias=bias)
    self.out4 = nn.Conv2d(inner_channels, inner_channels // 4, 3, padding=1, bias=bias)
    self.out3 = nn.Conv2d(inner_channels, inner_channels // 4, 3, padding=1, bias=bias)
    self.out2 = nn.Conv2d(inner_channels, inner_channels // 4, 3, padding=1, bias=bias)

    self.in5.apply(self.weights_init)
    self.in4.apply(self.weights_init)
    self.in3.apply(self.weights_init)
    self.in2.apply(self.weights_init)
    self.out5.apply(self.weights_init)
    self.out4.apply(self.weights_init)
    self.out3.apply(self.weights_init)
    self.out2.apply(self.weights_init)