builder

data.dataset.builder

DATASET_IMPLEMENTED module-attribute

DATASET_IMPLEMENTED = {'srproj': SrprojDataset}

SaigeDataset

SaigeDataset(crop: Optional[dict] = None, transform: Optional[dict] = None, collate: Optional[Callable[[List[dict]], dict]] = None, device: device = torch.device('cpu'), **neglect: dict)

Bases: Dataset

Saige's basic dataset class.

Attributes:

• crop_fn (Optional[Callable[[dict], List[dict]]) –

  function cropping image data into image patch data.

• transform (Type[Transform]) –

  transform class for transforming to a single image data.

• to_tensor (Callable[[Union[Image.Image, np.ndarray], torch.Tensor]) –

  data returning as Tensor type

• n_patch (int) –

  number of patches when crop_fn is used.

• collate_function (Callable[[List[dict]], dict]) –

  function list of data collating into one batch data

initializing Saige base dataset.

Parameters:

• crop (Optional[dict], default: None ) –

  config for crop function. Defaults to None.

• transform (Optional[dict], default: None ) –

  config for transforming to a single raw data

• collate (Optional[Callable[[List[dict]], dict]], default: None ) –

  list of data collating into one batch data. if None, torch.utils basic collate function would be used. Defaults to None.

• device (device, default: device('cpu') ) –

  gpu device

Source code in SaigeToolkit/data/dataset/base_dataset.py

def __init__(
    self,
    crop: Optional[dict] = None,
    transform: Optional[dict] = None,
    collate: Optional[Callable[[List[dict]], dict]] = None,
    device: torch.device = torch.device("cpu"),
    **neglect: dict,
) -> None:
    """initializing Saige base dataset.

    Args:
        crop (Optional[dict], optional):
            config for crop function. Defaults to None.
        transform (Optional[dict], optional):
            config for transforming to a single raw data
        collate (Optional[Callable[[List[dict]], dict]], optional):
            list of data collating into one batch data.
            if None, torch.utils basic collate function would be used. Defaults to None.
        device (torch.device, optional):
            gpu device
    """
    if neglect:
        logger.warning(f"Neglected arguments: {neglect}")

    # Config options
    self.crop_fn = get_crop_fn(**(crop or {}))
    self.has_crop_fn = crop is not None

    self.transform = Transform(**(transform or {}))

    self.device = device

    if collate is not None:
        assert collate in ["resize", "padding"]
        self.collate_function = {
            "resize": self.resize_collate,
            "padding": self.padding_collate,
        }[collate]
        logger.info(f'[{"DATA".center(9)}] [Base dataset collate] {collate}')

crop_fn instance-attribute

crop_fn = get_crop_fn(**(crop or {}))

has_crop_fn instance-attribute

has_crop_fn = crop is not None

transform instance-attribute

transform = Transform(**(transform or {}))

device instance-attribute

device = device

collate_function instance-attribute

collate_function = {'resize': resize_collate, 'padding': padding_collate}[collate]

files property writable

files

__len__

__len__()

Source code in SaigeToolkit/data/dataset/base_dataset.py

def __len__(self):
    return len(self.files)

data_on_memory

data_on_memory()

To store data in RAM memory (faster data loading), put all loaded data ( use self.load_raw_data() ) in list self.data_on_memory. Or define self.data_on_memory as empty list to load data from filesystem everytime.

Source code in SaigeToolkit/data/dataset/base_dataset.py

@abstract_attribute
def data_on_memory(self):
    """
    To store data in RAM memory (faster data loading),
    put all loaded data ( use self.load_raw_data() ) in list `self.data_on_memory`.
    Or define self.data_on_memory as empty list to load data from filesystem everytime.
    """
    pass

stack_data_on_memory

stack_data_on_memory() -> List[dict]

Stack all data on RAM memory for quick-loading purpose. output would be saved on class attribute.

Returns:

• List[dict] –

  List[dict]: Entire data list-dict.

Source code in SaigeToolkit/data/dataset/base_dataset.py

def stack_data_on_memory(self) -> List[dict]:
    """Stack all data on RAM memory for quick-loading purpose.
    output would be saved on class attribute.

    Returns:
        List[dict]: Entire data list-dict.
    """
    data_on_memory = []
    for idx in range(len(self)):
        data_on_memory.append(self.load_raw_data(idx))

    logger.info(f'[{"DATA".center(9)}] [DATA_ON_MEMORY] activated')

    return data_on_memory

load_raw_data

load_raw_data(index)

Source code in SaigeToolkit/data/dataset/base_dataset.py

def load_raw_data(self, index):
    # load image
    file = self.files[index]
    data = self.load_image(file)
    data.update(self.load_label(file))

    return data

load_image

load_image(file)

Source code in SaigeToolkit/data/dataset/base_dataset.py

def load_image(self, file):
    # open image
    if "image_path" in file:
        image = file["image_path"]
    elif "path" in file:
        image = file["path"]
    elif "Path" in file:
        image = file["Path"]
    else:
        raise NotImplementedError(f"no known image path item in {file.keys()}")

    data = self.transform.image_loader(image)

    return data

load_label abstractmethod

load_label(file)

Source code in SaigeToolkit/data/dataset/base_dataset.py

@abstractmethod
def load_label(self, file):
    pass

__getitem__

__getitem__(index: int) -> Union[dict, List[dict]]

dataset default getitem function. data is first loaded, then processed as following order: resize, augmentation, transform to torch.Tensor. data could be loaded from file_system (load_raw_data) or RAM memory (data_on_memory).

Parameters:

• index (int) –

  data item index.

Returns:

• Union[dict, List[dict]] –

  Union[dict, List[dict]]: single data, or cropped data list when crop_fn exists.

Source code in SaigeToolkit/data/dataset/base_dataset.py

def __getitem__(self, index: int) -> Union[dict, List[dict]]:
    """dataset default __getitem__ function.
    data is first loaded, then processed as following order:
        resize, augmentation, transform to torch.Tensor.
    data could be loaded from file_system (load_raw_data) or RAM memory (data_on_memory).

    Args:
        index (int): data item index.

    Returns:
        Union[dict, List[dict]]:
            single data, or cropped data list when crop_fn exists.
    """

    # Load Images and Text or Image Labels
    if self.data_on_memory:
        data = self.data_on_memory[index]
    else:
        data = self.load_raw_data(index)

    # ROI
    if self.transform.roi_handler is not None:
        if self.has_crop_fn:
            data = self.transform.roi_handler(**data)
        else:
            data = self.transform.roi_handler.apply_crop(**data)

    assert "image" in data, "SaigeDataset currently only supports image dataset"

    if not self.has_crop_fn:
        data = self.process_data(data)

        return data

    else:
        l_data_raw = self.crop_fn(**data)

        batch = []
        for data_patch in l_data_raw:
            batch.append(self.process_data(data_patch))

        return batch

process_data

process_data(data: dict) -> dict

data processing after raw loading.

- resize: Resize Image and Label
- btw_resize_aug: Dummy function for user customization
- augmentation: Process augmentation defined in util/augmentation
- btw_aug_transform: Dummy function for user customization
- to_tensor: Transform data to torch tensor

Parameters:

• data (dict) –

  raw data dict

Returns:

• dict ( dict ) –

  processed data dict

Source code in SaigeToolkit/data/dataset/base_dataset.py

def process_data(self, data: dict) -> dict:
    """data processing after raw loading.

        - resize: Resize Image and Label
        - btw_resize_aug: Dummy function for user customization
        - augmentation: Process augmentation defined in util/augmentation
        - btw_aug_transform: Dummy function for user customization
        - to_tensor: Transform data to torch tensor

    Args:
        data (dict): raw data dict

    Returns:
        dict: processed data dict
    """
    if self.transform.base_resizer is not None:
        data = self.transform.base_resizer(**data)

    if self.transform.roi_handler is not None and not self.has_crop_fn:
        data = self.transform.roi_handler.apply_mask(**data)

    if self.transform.augmentation is not None:
        data = self.transform.augmentation(**data)

    data = self.to_tensor(**data)

    return data

to_tensor

to_tensor(**data: dict) -> dict

to_tensor items in data into torch.Tensor (if convertible)

Parameters:

• data (dict, default: {} ) –

  raw data dict

Returns:

• dict ( dict ) –

  tensorized data dict

Source code in SaigeToolkit/data/dataset/base_dataset.py

def to_tensor(self, **data: dict) -> dict:
    """to_tensor items in data into torch.Tensor (if convertible)

    Args:
        data (dict): raw data dict

    Returns:
        dict: tensorized data dict
    """

    def _to_tensor_from_numpy(image: np.ndarray):
        return torch.from_numpy(image).to(self.device).contiguous()

    def _to_tensor_from_pil_image(image: Image.Image):
        return _to_tensor_from_numpy(np.array(image))

    for k, v in data.items():
        if k == "org_image":
            continue

        if isinstance(v, Image.Image):
            data[k] = _to_tensor_from_pil_image(v)

        elif isinstance(v, np.ndarray):
            data[k] = _to_tensor_from_numpy(v)

        elif isinstance(v, int):
            data[k] = torch.tensor(v).to(self.device).contiguous()

        else:
            continue

        if k == "image":
            data[k] = data[k].permute((2, 0, 1)).div(255)
        else:
            data[k] = data[k].long()

    return data

resize_collate

resize_collate(batch: List[dict]) -> dict

Example collate function with resizing items.

Parameters:

• batch (List[dict]) –

  list of data dicts

Returns:

• dict ( dict ) –

  resized-and-collated data dict

Source code in SaigeToolkit/data/dataset/base_dataset.py

def resize_collate(self, batch: List[dict]) -> dict:
    """Example collate function with resizing items.

    Args:
        batch (List[dict]): list of data dicts

    Returns:
        dict: resized-and-collated data dict
    """

    img_size = (
        int(np.mean([data["image"].shape[-2] for data in batch])),
        int(np.mean([data["image"].shape[-1] for data in batch])),
    )

    # 이미 tensor 형태로 들어온 상태기 때문에 F.interpolate 함수 사용해보자
    data_out = {key: [] for key in batch[0].keys()}
    for data in batch:
        data_out["image"].append(F.interpolate(data["image"][None], (img_size[0], img_size[1])))

        for k, v in data.items():
            if k == "class":
                data_out[k].append(v.unsqueeze(0))
            elif k == "boxes":
                data_out[k].append(v)
            elif k == "mask":
                data_out[k].append(
                    F.interpolate(v[None, None].float(), (img_size[0], img_size[1]))
                    .long()
                    .squeeze(1)
                )

    for key, value in data_out.items():
        data_out[key] = torch.cat(value)

    return data_out

padding_collate

padding_collate(batch: List[dict]) -> dict

Example collate function with padding items.

Parameters:

• batch (List[dict]) –

  list of data dicts

Returns:

• dict ( dict ) –

  padded-and-collated data dict

Source code in SaigeToolkit/data/dataset/base_dataset.py

def padding_collate(self, batch: List[dict]) -> dict:
    """Example collate function with padding items.

    Args:
        batch (List[dict]): list of data dicts

    Returns:
        dict: padded-and-collated data dict
    """

    max_h = np.max([d["image"].shape[-2] for d in batch])
    max_w = np.max([d["image"].shape[-1] for d in batch])

    data_out = {key: [] for key in batch[0].keys()}
    for data in batch:
        data_out["image"].append(
            F.pad(
                data["image"][None],
                (0, max_w - data["image"].shape[-1], 0, max_h - data["image"].shape[-2]),
            )
        )

        for k, v in data.items():
            if k == "class":
                data_out[k].append(v.unsqueeze(0))
            elif k == "boxes":
                data_out[k].append(v)
            elif k == "mask":
                data_out[k].append(F.pad(v[None], (0, max_w - v.shape[-1], 0, max_h - v.shape[-2])))

    for key, value in data_out.items():
        data_out[key] = torch.cat(value)

    return data_out

calculate_patch_number

calculate_patch_number(files: Optional[List[dict]] = None) -> int

When crop_fn is activated, calculate number of text patches.

Parameters:

• files (Optional[List[dict]], default: None ) –

  list of meta data dict. if None, calculate patch_number from self.files attribute. Defaults to None.

Returns:

• int ( int ) –

  number of patches in meta data list files

Source code in SaigeToolkit/data/dataset/base_dataset.py

def calculate_patch_number(self, files: Optional[List[dict]] = None) -> int:
    """When crop_fn is activated, calculate number of text patches.

    Args:
        files (Optional[List[dict]], optional): list of meta data dict.
            if None, calculate patch_number from self.files attribute. Defaults to None.

    Returns:
        int: number of patches in meta data list `files`
    """
    if not self.has_crop_fn:
        return len(self)

    if files is None:
        files = self.files

    n_patch = 0
    for file in files:
        labels = self.load_label(file)
        if "polygons" in labels:
            n_patch += self.crop_fn.get_n_patch(labels["polygons"])
        elif "labels" in labels:
            n_patch += len(labels["labels"])
        elif "labels_count" in labels:
            n_patch += labels["labels_count"]
        else:
            n_patch += 1

    logger.info(f"total patch number in files: {n_patch}")

    return n_patch

SrprojDataset

SrprojDataset(code: Union[str, List[Union[str, dict]]], mode: Optional[str] = None, two_class: bool = False, split: str = 'Training', preload_data_on_memory: bool = False, srproj_params: Optional[dict] = None, **cfg_dataset: dict)

Bases: SaigeDataset

Dataset class with building data from srproj file. Basic structure of dataset class is defined in SaigeDataset. SrprojDataset class only defines data loading from srproj file.

Attributes:

• files (List[dict]) –

  meta_data dict list from srproj file.

• n_classes (int) –

  number of classes of data. fix to 2 if two_class mode is on.

• classes (List[str]) –

  list of class names for each class index. meta_data: only contains information about how to load data

• n_patch (int) –

  number of patches when crop_fn is used.

• data_on_memory (List[dict]) –

  list of loaded data dicts if preload_data_on_memory is activated

initializing SrprojDataset

Parameters:

• code (Union[str, List[Union[str, dict]]]) –

  dataset code(s) with srproj params. 여러 srproj를 사용하는 경우, 각 srproj_param을 dictionary 형태로 추가할 수 있습니다. 이 경우, "srproj_params" 파라미터를 각자의 srproj_param으로 업데이트 하여 사용합니다.

• mode (Optional[str], default: None ) –

  data label type. Defaults to None.

• two_class (bool, default: False ) –

  two_class mode selector. Defaults to False.

• split (str, default: 'Training' ) –

  ["Training", "Validation"]. Defaults to "Training".

• preload_data_on_memory (bool, default: False ) –

  whether pre-load all data and save on RAM memory. Defaults to False.

• srproj_params (dict, default: None ) –

  read_srproj 함수에 전달하는 추가적인 파라미터들 입니다. 현재는 이미지 경로 핸들링을 위한 파라미터들이 있으며, 지속적으로 추가될 수 있습니다.

• cfg_dataset (dict, default: {} ) –

  config dicts for mother class

Source code in SaigeToolkit/data/dataset/srproj_dataset.py

def __init__(
    self,
    code: Union[str, List[Union[str, dict]]],
    mode: Optional[str] = None,
    two_class: bool = False,
    split: str = "Training",
    preload_data_on_memory: bool = False,
    srproj_params: Optional[dict] = None,
    **cfg_dataset: dict,
) -> None:
    """initializing SrprojDataset

    Args:
        code (Union[str, List[Union[str, dict]]]): dataset code(s) with srproj params.
                                                여러 srproj를 사용하는 경우, 각 srproj_param을 dictionary 형태로 추가할 수 있습니다.
                                                이 경우, "srproj_params" 파라미터를 각자의 srproj_param으로 업데이트 하여 사용합니다.
        mode (Optional[str], optional): data label type. Defaults to None.
        two_class (bool, optional): two_class mode selector. Defaults to False.
        split (str, optional): ["Training", "Validation"]. Defaults to "Training".
        preload_data_on_memory (bool, optional):
            whether pre-load all data and save on RAM memory. Defaults to False.
        srproj_params (dict, optional): `read_srproj` 함수에 전달하는 추가적인 파라미터들 입니다.
                                        현재는 이미지 경로 핸들링을 위한 파라미터들이 있으며, 지속적으로 추가될 수 있습니다.
        cfg_dataset (dict): config dicts for mother class
    """
    super().__init__(**cfg_dataset)

    if srproj_params is None:
        srproj_params = {}

    if isinstance(code, list):
        self.files = []
        self.classes = []
        for c in code:
            if isinstance(c, dict):
                _c = c.pop("code")
                _srproj_params = deepcopy(srproj_params)
                _srproj_params.update(c)
                files, _, classes = read_srproj(_c, split, two_class, mode, **_srproj_params)
            elif isinstance(c, str):
                files, _, classes = read_srproj(c, split, two_class, mode, **srproj_params)
            else:
                raise NotImplementedError

            self.files.extend(files)
            self.classes = merge_srproj_classes(self.classes, classes)
        self.n_classes = len(self.classes)
    else:
        self.files, self.n_classes, self.classes = read_srproj(
            code, split, two_class, mode, **srproj_params
        )

    self.data_on_memory = self.stack_data_on_memory() if preload_data_on_memory else []

    logger.info(
        f'[{"DATA".center(9)}] [count {split}] {len(self)} [number of classes] {self.n_classes}'
    )

files instance-attribute

files = []

classes instance-attribute

classes = merge_srproj_classes(classes, classes)

n_classes instance-attribute

n_classes = len(classes)

data_on_memory instance-attribute

data_on_memory = stack_data_on_memory() if preload_data_on_memory else []

load_label

load_label(file: dict) -> dict

load label data from file-system

Parameters:

• file (dict) –

  meta data dict

Returns:

• dict ( dict ) –

  label data as dict

Source code in SaigeToolkit/data/dataset/srproj_dataset.py

def load_label(self, file: dict) -> dict:
    """load label data from file-system

    Args:
        file (dict): meta data dict

    Returns:
        dict: label data as dict
    """
    if self.crop_fn is not None:
        return load_srproj_label(file, use_crop_fn=True)
    else:
        return load_srproj_label(file, use_crop_fn=False)

build_dataset

build_dataset(_target_, **cfg_dataset: dict) -> SaigeDataset

build dataset from config["dataset"]

Parameters:

• cfg_dataset (dict, default: {} ) –

  config dict for building dataset

Returns:

• SaigeDataset ( SaigeDataset ) –

  dataset object

Source code in SaigeToolkit/data/dataset/builder.py

def build_dataset(_target_, **cfg_dataset: dict) -> SaigeDataset:
    """build dataset from config["dataset"]

    Args:
        cfg_dataset (dict): config dict for building dataset

    Returns:
        SaigeDataset: dataset object
    """

    # Setup Dataset
    dataset_class = DATASET_IMPLEMENTED[_target_]
    dataset: SaigeDataset = dataset_class(**cfg_dataset)

    return dataset