Skip to content

ocr_cropper

data.crop.ocr_cropper

logger module-attribute 

logger = getLogger('SaigeResearch')

OcrCropper 

OcrCropper(chr_height: int = 32, crop_jitter: Optional[float] = None, crop_jitter_rate: Optional[List[float]] = None, save_polygons: bool = False)

Cropper class for OCR task.

Attributes:

  • chr_height (int) –

    cropped image patch is resized to be the same height as chr_height.

  • crop_jitter (Optional[float]) –

    if not None, cropping box is jittered horizontally with ratio crop_jitter.

  • crop_jitter_rate (Optional[List[float]]) –

    if not None, cropping box points are jittered to any direction with ratio crop_jitter_rate.

  • save_polygons (bool) –

    whether polygon label is preserved after cropping.

initializing OcrCropper. all initializing input is set to class attribute.

Parameters:

  • chr_height (int, default: 32 ) –

    Defaults to 32.

  • crop_jitter (Optional[float], default: None ) –

    Defaults to None.

  • crop_jitter_rate (Optional[List[float]], default: None ) –

    Defaults to None.

  • save_polygons (bool, default: False ) –

    Defaults to False.

Source code in SaigeToolkit/data/crop/ocr_cropper.py 
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
def __init__(
    self,
    chr_height: int = 32,
    crop_jitter: Optional[float] = None,
    crop_jitter_rate: Optional[List[float]] = None,
    save_polygons: bool = False,
) -> None:
    """initializing OcrCropper.
    all initializing input is set to class attribute.

    Args:
        chr_height (int): Defaults to 32.
        crop_jitter (Optional[float], optional): Defaults to None.
        crop_jitter_rate (Optional[List[float]], optional): Defaults to None.
        save_polygons (bool): Defaults to False.
    """
    self.chr_height = chr_height
    self.crop_jitter = crop_jitter
    self.crop_jitter_rate = crop_jitter_rate
    self.save_polygons = save_polygons

get_n_patch 

get_n_patch(polygons: List[List[ndarray]]) -> int

OcrCropper only crops all polygon areas, unlike SegCropper. Therefore, resultant number of patches is simply equals to number of polygons.

Parameters:

  • polygons (List[List[ndarray]]) –

    polygons of all images in dataset.

Returns:

  • int ( int ) –

    total number of patches to be cropped

Source code in SaigeToolkit/data/crop/ocr_cropper.py 
47
48
49
50
51
52
53
54
55
56
57
def get_n_patch(self, polygons: List[List[np.ndarray]]) -> int:
    """OcrCropper only crops all polygon areas, unlike SegCropper.
    Therefore, resultant number of patches is simply equals to number of polygons.

    Args:
        polygons (List[List[np.ndarray]]): polygons of all images in dataset.

    Returns:
        int: total number of patches to be cropped
    """
    return len(polygons)

__call__ 

__call__(image: Union[Image, ndarray], polygons: List[ndarray], strings: List[str], ignore: List[bool], is_vertical: Optional[List[bool]] = None, **kwargs) -> List[dict]

crop image into image patches. polygon, string, ignore data should be same length.

Parameters:

  • image (Union[Image, ndarray]) –

    original image

  • polygons (List[ndarray]) –

    polygon data (# of polygons, (4, 2)) polygon should have 4 points

  • strings (List[str]) –

    string data

  • ignore (List[bool]) –

    ignore notation data

  • is_vertical (List[bool], default: None ) –

    is_vertical notation data

Returns:

  • List[dict] –

    List[dict]: list of cropped image data dict

Source code in SaigeToolkit/data/crop/ocr_cropper.py 
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
def __call__(
    self,
    image: Union[Image.Image, np.ndarray],
    polygons: List[np.ndarray],
    strings: List[str],
    ignore: List[bool],
    is_vertical: Optional[List[bool]] = None,
    **kwargs,
) -> List[dict]:
    """crop image into image patches.
    polygon, string, ignore data should be same length.

    Args:
        image (Union[Image.Image, np.ndarray]): original image
        polygons (List[np.ndarray]): polygon data
            (# of polygons, (4, 2)) polygon should have 4 points
        strings (List[str]): string data
        ignore (List[bool]): ignore notation data
        is_vertical (List[bool]): is_vertical notation data

    Returns:
        List[dict]: list of cropped image data dict
    """
    image = np.asarray(image)

    if is_vertical is None:
        is_vertical = [False] * len(polygons)

    l_data = []
    for pol, string, ign, is_vert in zip(polygons, strings, ignore, is_vertical):
        if ign:
            continue

        max_trial = 10
        trial = 0
        while trial < max_trial:
            trial += 1

            tops, bots = self.jitter_polygon(pol)
            widths, heights = self.calculate_width_height(tops, bots)

            if all(np.array(widths + heights) >= 1):
                break

            if trial == max_trial:
                logger.info("Max trial exeeded.. no jitter applied.")
                tops = pol[: len(pol) // 2]
                bots = pol[: len(pol) // 2 - 1 : -1]
                widths, heights = self.calculate_width_height(tops, bots)

        if any(np.array(widths + heights) < 1):
            logger.warning(f"Invalid patch shape (jittered) encounter.. string: {string}")
            continue

        text_patch_list = []
        for i, (w, h) in enumerate(zip(widths, heights)):
            width = int(np.round((w / h * self.chr_height)))
            if width == 0:
                continue

            startpoints = np.array([tops[i], tops[i + 1], bots[i + 1], bots[i]], np.float32)
            endpoints = np.array(
                [
                    [0, 0],
                    [width - 1, 0],
                    [width - 1, self.chr_height - 1],
                    [0, self.chr_height - 1],
                ],
                np.float32,
            )

            affine_matrix = cv2.getPerspectiveTransform(startpoints, endpoints)
            text_patch = cv2.warpPerspective(
                image, affine_matrix, (width, self.chr_height), borderValue=0
            )
            if i != 0:
                # remove duplicated leftmost column
                # border area is duplicated in adjacent patches.
                text_patch = text_patch[:, 1:, :]
            text_patch_list.append(text_patch)

        text_patch = np.concatenate(text_patch_list, axis=1)

        if int(text_patch.shape[1] / (self.chr_height / 8)) + 1 >= len(string):
            patch_data = {
                "image": Image.fromarray(text_patch),
                "strings": string,
                "is_vertical": is_vert,
            }

            if self.save_polygons:
                patch_data.update(polygons=pol)

            l_data.append(patch_data)
        else:
            logger.warning(
                f"Invalid patch length! string: {string} patch width: {text_patch.shape[1]}"
            )

    return l_data

calculate_width_height 

calculate_width_height(tops: ndarray, bots: ndarray) -> Tuple[List[ndarray], List[ndarray]]

calculate width and heights for every four polygon points (2 tops and 2 bots). Assuming that polygon is composed of top-points and bot-points, and every top and bot points are paired.

Parameters:

  • tops (ndarray) –

    top points of polygon

  • bots (ndarray) –

    bot points of polygon

Returns:

  • Tuple[List[ndarray], List[ndarray]] –

    Tuple[List[np.ndarray]]: width and height list for every four points.

Source code in SaigeToolkit/data/crop/ocr_cropper.py 
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
def calculate_width_height(
    self, tops: np.ndarray, bots: np.ndarray
) -> Tuple[List[np.ndarray], List[np.ndarray]]:
    """calculate width and heights for every four polygon points (2 tops and 2 bots).
    Assuming that polygon is composed of top-points and bot-points,
    and every top and bot points are paired.

    Args:
        tops (np.ndarray): top points of polygon
        bots (np.ndarray): bot points of polygon

    Returns:
        Tuple[List[np.ndarray]]: width and height list for every four points.
    """

    widths = []
    for i in range(len(tops) - 1):
        widths.append(
            np.mean(
                (
                    np.linalg.norm(tops[i + 1] - tops[i]),
                    np.linalg.norm(bots[i + 1] - bots[i]),
                )
            )
        )

    heights = []
    for t, b in zip(tops, bots):
        heights.append(np.linalg.norm(b - t))
    heights = [np.mean([heights[i + 1], heights[i]]) for i in range(len(heights) - 1)]

    return widths, heights

jitter_polygon 

jitter_polygon(polygon: ndarray) -> Tuple[ndarray, ndarray]

jittering polygon points and split into tops and bots points

Parameters:

  • polygon (ndarray) –

    original polygon data

Returns:

  • Tuple[ndarray, ndarray] –

    Tuple[np.ndarray, np.ndarray]: jittered tops, bots points

Source code in SaigeToolkit/data/crop/ocr_cropper.py 
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
def jitter_polygon(self, polygon: np.ndarray) -> Tuple[np.ndarray, np.ndarray]:
    """jittering polygon points and split into tops and bots points

    Args:
        polygon (np.ndarray): original polygon data

    Returns:
        Tuple[np.ndarray, np.ndarray]: jittered tops, bots points
    """
    tops = deepcopy(polygon[: len(polygon) // 2])
    bots = deepcopy(polygon[: len(polygon) // 2 - 1 : -1])

    if self.crop_jitter is not None:

        heights = []
        for i, (t, b) in enumerate(zip(tops, bots)):
            heights.append(np.linalg.norm(b - t))

            crop_jitter = self.crop_jitter * heights[-1]

            x_jitter = random.uniform(-crop_jitter, crop_jitter)
            tops[i, 0] += x_jitter
            bots[i, 0] += x_jitter

        height = np.mean(heights)
        y_jitter = self.crop_jitter * height

        tops[:, 1] += random.uniform(-y_jitter, y_jitter)
        bots[:, 1] += random.uniform(-y_jitter, y_jitter)

    elif self.crop_jitter_rate is not None:

        for i, (t, b) in enumerate(zip(tops, bots)):
            height = np.linalg.norm(b - t)

            crop_jitter_short = self.crop_jitter_rate[0] * height
            crop_jitter_long = self.crop_jitter_rate[1] * height

            top_y_jitter = random.uniform(-crop_jitter_long, crop_jitter_short)
            bot_y_jitter = random.uniform(-crop_jitter_short, crop_jitter_long)
            if i == 0:
                x_jitter = random.uniform(-crop_jitter_long, crop_jitter_short)
            elif i == len(tops) - 1:
                x_jitter = random.uniform(-crop_jitter_short, crop_jitter_long)
            else:
                x_jitter = random.uniform(-crop_jitter_short, crop_jitter_short)

            tops[i, :] += (x_jitter, top_y_jitter)
            bots[i, :] += (x_jitter, bot_y_jitter)

    # polygon_out = np.concatenate((tops, bots[::-1]), axis=0)

    return tops, bots